[llvm] 96ca8cf - Revert "[MachineLICM] Use `RegisterClassInfo::getRegPressureSetLimit`" (#213287)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 08:13:13 PDT 2026
Author: Pengcheng Wang
Date: 2026-07-31T23:12:54+08:00
New Revision: 96ca8cf60e0d2ee992f714dbe6ebc4cd079e50a7
URL: https://github.com/llvm/llvm-project/commit/96ca8cf60e0d2ee992f714dbe6ebc4cd079e50a7
DIFF: https://github.com/llvm/llvm-project/commit/96ca8cf60e0d2ee992f714dbe6ebc4cd079e50a7.diff
LOG: Revert "[MachineLICM] Use `RegisterClassInfo::getRegPressureSetLimit`" (#213287)
Reverts llvm/llvm-project#211715 because of some failures.
Added:
Modified:
llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
llvm/lib/CodeGen/MachineLICM.cpp
llvm/lib/CodeGen/OptimizePHIs.cpp
llvm/lib/CodeGen/StackColoring.cpp
llvm/lib/Target/RISCV/RISCVFoldMemOffset.cpp
llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/irreducible/diverged-entry-basic-gmir.mir
llvm/test/Analysis/UniformityAnalysis/AMDGPU/irreducible/diverged-entry-basic.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
llvm/test/CodeGen/AMDGPU/frem.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
llvm/test/CodeGen/AMDGPU/indirect-call.ll
llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
llvm/test/CodeGen/AMDGPU/memset-pattern.ll
llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
llvm/test/CodeGen/AMDGPU/valu-i1.ll
llvm/test/CodeGen/ARM/O3-pipeline.ll
llvm/test/CodeGen/LoongArch/jr-without-ra.ll
llvm/test/CodeGen/LoongArch/opt-pipeline.ll
llvm/test/CodeGen/RISCV/O3-pipeline.ll
llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
llvm/test/CodeGen/SPIRV/llc-pipeline.ll
llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
index a388413525c23..e77bae11dea46 100644
--- a/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/llvm/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -24,7 +24,6 @@
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineOperand.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -109,7 +108,6 @@ namespace {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
@@ -124,7 +122,6 @@ LocalStackSlotAllocationPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
- PA.preserve<MachineRegisterClassAnalysis>();
return PA;
}
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index f1ace5551a9fc..4a36b90d4881e 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -123,7 +123,6 @@ namespace {
const TargetRegisterInfo *TRI = nullptr;
const MachineFrameInfo *MFI = nullptr;
MachineRegisterInfo *MRI = nullptr;
- const RegisterClassInfo *RegClassInfo = nullptr;
TargetSchedModel SchedModel;
bool PreRegAlloc = false;
bool HasProfileData = false;
@@ -303,7 +302,6 @@ namespace {
if (DisableHoistingToHotterBlocks != UseBFI::None)
AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
AU.addRequired<MachineDominatorTreeWrapperPass>();
- AU.addRequired<MachineRegisterClassInfoWrapperPass>();
AU.addRequired<AAResultsWrapperPass>();
AU.addPreserved<MachineLoopInfoWrapperPass>();
AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
@@ -336,7 +334,6 @@ INITIALIZE_PASS_BEGIN(MachineLICM, DEBUG_TYPE,
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MachineRegisterClassInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_END(MachineLICM, DEBUG_TYPE,
"Machine Loop Invariant Code Motion", false, false)
@@ -346,7 +343,6 @@ INITIALIZE_PASS_BEGIN(EarlyMachineLICM, "early-machinelicm",
INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
-INITIALIZE_PASS_DEPENDENCY(MachineRegisterClassInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_END(EarlyMachineLICM, "early-machinelicm",
"Early Machine Loop Invariant Code Motion", false, false)
@@ -370,13 +366,6 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
.getManager()
.getResult<AAManager>(MF.getFunction())
: &LegacyPass->getAnalysis<AAResultsWrapperPass>().getAAResults();
-
- RegClassInfo =
- MFAM != nullptr
- ? &MFAM->getResult<MachineRegisterClassAnalysis>(MF)
- : &LegacyPass->getAnalysis<MachineRegisterClassInfoWrapperPass>()
- .getRCI();
-
MachineDomTreeUpdater DTU(GET_RESULT(MachineDominatorTree, getDomTree, ),
MachineDomTreeUpdater::UpdateStrategy::Lazy);
MDTU = &DTU;
@@ -409,7 +398,7 @@ bool MachineLICMImpl::run(MachineFunction &MF) {
llvm::fill(RegPressure, 0);
RegLimit.resize(NumRPS);
for (unsigned i = 0, e = NumRPS; i != e; ++i)
- RegLimit[i] = RegClassInfo->getRegPressureSetLimit(i);
+ RegLimit[i] = TRI->getRegPressureSetLimit(MF, i);
}
if (HoistConstLoads)
diff --git a/llvm/lib/CodeGen/OptimizePHIs.cpp b/llvm/lib/CodeGen/OptimizePHIs.cpp
index ff38bc28268e4..c27f2c4cc577c 100644
--- a/llvm/lib/CodeGen/OptimizePHIs.cpp
+++ b/llvm/lib/CodeGen/OptimizePHIs.cpp
@@ -19,7 +19,6 @@
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
@@ -65,7 +64,6 @@ class OptimizePHIsLegacy : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
};
@@ -85,7 +83,6 @@ PreservedAnalyses OptimizePHIsPass::run(MachineFunction &MF,
return PreservedAnalyses::all();
auto PA = getMachineFunctionPassPreservedAnalyses();
PA.preserveSet<CFGAnalyses>();
- PA.preserve<MachineRegisterClassAnalysis>();
return PA;
}
diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp
index aa42c0ee28532..ea1f33f791765 100644
--- a/llvm/lib/CodeGen/StackColoring.cpp
+++ b/llvm/lib/CodeGen/StackColoring.cpp
@@ -38,7 +38,6 @@
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/CodeGen/PseudoSourceValueManager.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/SlotIndexes.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
@@ -526,7 +525,6 @@ INITIALIZE_PASS_END(StackColoringLegacy, DEBUG_TYPE,
void StackColoringLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
AU.addRequired<SlotIndexesWrapperPass>();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -1196,11 +1194,8 @@ bool StackColoringLegacy::runOnMachineFunction(MachineFunction &MF) {
PreservedAnalyses StackColoringPass::run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM) {
StackColoring SC(&MFAM.getResult<SlotIndexesAnalysis>(MF));
- if (SC.run(MF)) {
- auto PA = getMachineFunctionPassPreservedAnalyses();
- PA.preserve<MachineRegisterClassAnalysis>();
- return PA;
- }
+ if (SC.run(MF))
+ return getMachineFunctionPassPreservedAnalyses();
return PreservedAnalyses::all();
}
diff --git a/llvm/lib/Target/RISCV/RISCVFoldMemOffset.cpp b/llvm/lib/Target/RISCV/RISCVFoldMemOffset.cpp
index 46d9fee2c9572..d325dd322b34d 100644
--- a/llvm/lib/Target/RISCV/RISCVFoldMemOffset.cpp
+++ b/llvm/lib/Target/RISCV/RISCVFoldMemOffset.cpp
@@ -18,7 +18,6 @@
#include "RISCV.h"
#include "RISCVSubtarget.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include <queue>
using namespace llvm;
@@ -42,7 +41,6 @@ class RISCVFoldMemOffset : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
diff --git a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
index f4a70ba4e52fb..8e038890d8347 100644
--- a/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVLOptimizer.cpp
@@ -32,7 +32,6 @@
#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/InitializePasses.h"
using namespace llvm;
@@ -78,7 +77,6 @@ class RISCVVLOptimizer : public MachineFunctionPass {
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.setPreservesCFG();
AU.addRequired<MachineDominatorTreeWrapperPass>();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
MachineFunctionPass::getAnalysisUsage(AU);
}
diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index 2f1b3c409ff2b..6071006882b31 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -31,7 +31,6 @@
#include "RISCVSubtarget.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -59,12 +58,6 @@ class RISCVVectorPeephole : public MachineFunctionPass {
return "RISC-V Vector Peephole Optimization";
}
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
- AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
- MachineFunctionPass::getAnalysisUsage(AU);
- }
-
private:
bool convertToVLMAX(MachineInstr &MI) const;
bool convertToWholeRegister(MachineInstr &MI) const;
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/irreducible/diverged-entry-basic-gmir.mir b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/irreducible/diverged-entry-basic-gmir.mir
index e3233b7ef6381..b6de0d6ec093d 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/irreducible/diverged-entry-basic-gmir.mir
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/MIR/irreducible/diverged-entry-basic-gmir.mir
@@ -4,8 +4,8 @@
# CHECK-NEXT: CYCLES ASSUMED DIVERGENT:
# CHECK-NEXT: depth=1: entries(bb.3 bb.1) bb.4 bb.2
# CHECK-NEXT: CYCLES WITH DIVERGENT EXIT:
-# CHECK-NEXT: depth=1: entries(bb.3 bb.1) bb.4 bb.2
# CHECK-NEXT: depth=2: entries(bb.4 bb.1) bb.2
+# CHECK-NEXT: depth=1: entries(bb.3 bb.1) bb.4 bb.2
diff --git a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/irreducible/diverged-entry-basic.ll b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/irreducible/diverged-entry-basic.ll
index 4f253ca0378f1..033ecac072cbe 100644
--- a/llvm/test/Analysis/UniformityAnalysis/AMDGPU/irreducible/diverged-entry-basic.ll
+++ b/llvm/test/Analysis/UniformityAnalysis/AMDGPU/irreducible/diverged-entry-basic.ll
@@ -5,8 +5,8 @@ define amdgpu_kernel void @divergent_cycle_1(i32 %a, i32 %b, i32 %c) {
; CHECK: CYCLES ASSUMED DIVERGENT:
; CHECK: depth=1: entries(R P) S Q
; CHECK: CYCLES WITH DIVERGENT EXIT:
-; CHECK: depth=1: entries(R P) S Q
; CHECK: depth=2: entries(S P) Q
+; CHECK: depth=1: entries(R P) S Q
entry:
%cond.uni = icmp slt i32 %a, 0
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
index c7f2ec5d96663..d62546e3de255 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmax.ll
@@ -325,13 +325,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX942-NEXT: v_max_f32_e32 v4, v3, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
@@ -371,13 +371,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -395,13 +395,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -470,22 +470,22 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v3, v4
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -515,20 +515,20 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v3, v4
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -538,20 +538,20 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -604,15 +604,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -642,15 +642,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -688,15 +688,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -714,15 +714,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -760,21 +760,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -797,22 +797,22 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -842,21 +842,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -866,21 +866,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -920,13 +920,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX942-NEXT: v_max_f32_e32 v4, v3, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
@@ -966,13 +966,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v3, v2
; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -990,13 +990,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1061,22 +1061,22 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v3, v4
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1108,20 +1108,20 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v3, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1131,20 +1131,20 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1194,15 +1194,15 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1232,15 +1232,15 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1278,15 +1278,15 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1307,15 +1307,15 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -1348,21 +1348,21 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1385,22 +1385,22 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -1431,21 +1431,21 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1455,24 +1455,24 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1508,25 +1508,23 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v0
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_mov_b32_e32 v0, s6
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v1, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX942-NEXT: v_max_f32_e32 v4, v0, v3
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
@@ -1562,22 +1560,20 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: v_mov_b32_e32 v0, s6
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v1, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v3
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
@@ -1591,26 +1587,24 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v0
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: v_mov_b32_e32 v0, s6
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v3
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
@@ -1621,25 +1615,24 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_m
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
@@ -1677,28 +1670,27 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v3, s6
+; GFX942-NEXT: v_mov_b32_e32 v1, s6
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v1, v3, v3
-; GFX942-NEXT: v_max_f32_e32 v2, v0, v0
-; GFX942-NEXT: v_max_f32_e32 v2, v1, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1730,25 +1722,24 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v1, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v0, v0
-; GFX90A-NEXT: v_max_f32_e32 v2, v1, v2
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1758,26 +1749,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1787,26 +1777,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1833,32 +1822,31 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
-; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], null offen
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[2:3]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1881,33 +1869,31 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
-; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen glc
+; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1939,33 +1925,30 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
-; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v0
-; GFX908-NEXT: v_readfirstlane_b32 s7, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, s7
-; GFX908-NEXT: v_mov_b32_e32 v8, s6
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s6
+; GFX908-NEXT: v_mov_b32_e32 v5, s7
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1975,33 +1958,30 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
-; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v0
-; GFX8-NEXT: v_readfirstlane_b32 s7, v1
-; GFX8-NEXT: v_mov_b32_e32 v9, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2028,31 +2008,30 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s16
-; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen
+; GFX12-NEXT: v_mov_b32_e32 v6, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], null offen
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s4, v2
; GFX12-NEXT: v_readfirstlane_b32 s5, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2075,32 +2054,30 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s16
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen
+; GFX11-NEXT: v_mov_b32_e32 v6, s16
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s4, v2
; GFX11-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX11-NEXT: v_mov_b32_e32 v6, v2
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], 0 offen glc
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2132,31 +2109,30 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v6, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s6, v2
; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v4, s6
-; GFX908-NEXT: v_mov_b32_e32 v5, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v3, s7
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v9, v5
-; GFX908-NEXT: v_mov_b32_e32 v8, v4
-; GFX908-NEXT: v_mov_b32_e32 v7, v3
-; GFX908-NEXT: v_mov_b32_e32 v6, v2
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v6
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v7
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2166,31 +2142,30 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v6, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v2
; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v8, v4
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v6
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v7
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
index d06c52c2d5211..0530f5e69456b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_fmin.ll
@@ -325,13 +325,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX942-NEXT: v_min_f32_e32 v4, v3, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
@@ -371,13 +371,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -395,13 +395,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -470,22 +470,22 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX942-NEXT: v_min_f32_e32 v4, v3, v4
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -515,20 +515,20 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: v_min_f32_e32 v4, v3, v4
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -538,20 +538,20 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -604,15 +604,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -642,15 +642,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -688,15 +688,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -714,15 +714,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -760,21 +760,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -797,22 +797,22 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -842,21 +842,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -866,21 +866,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -920,13 +920,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX942-NEXT: v_min_f32_e32 v4, v3, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
@@ -966,13 +966,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v3
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v3, v2
; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -990,13 +990,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1061,22 +1061,22 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX942-NEXT: v_min_f32_e32 v4, v3, v4
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1108,20 +1108,20 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
-; GFX90A-NEXT: v_min_f32_e32 v4, v3, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1131,20 +1131,20 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1194,15 +1194,15 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -1232,15 +1232,15 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1278,15 +1278,15 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1307,15 +1307,15 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -1348,21 +1348,21 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -1385,22 +1385,22 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -1431,21 +1431,21 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1455,24 +1455,24 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v6, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v5, v[5:6]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1508,25 +1508,23 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s6, v0
+; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_mov_b32_e32 v0, s6
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX942-NEXT: v_max_f32_e32 v1, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v1, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX942-NEXT: v_min_f32_e32 v4, v0, v3
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
@@ -1562,22 +1560,20 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v0
+; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
; GFX90A-NEXT: v_mov_b32_e32 v0, s6
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX90A-NEXT: v_max_f32_e32 v1, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v1, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v3
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
@@ -1591,26 +1587,24 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v0
+; GFX908-NEXT: v_readfirstlane_b32 s6, v1
; GFX908-NEXT: v_mov_b32_e32 v0, s6
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v1, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v3
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
@@ -1621,25 +1615,24 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_m
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
@@ -1677,28 +1670,27 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v3, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_readfirstlane_b32 s6, v1
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_mov_b32_e32 v3, s6
+; GFX942-NEXT: v_mov_b32_e32 v1, s6
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v1, v3, v3
-; GFX942-NEXT: v_max_f32_e32 v2, v0, v0
-; GFX942-NEXT: v_min_f32_e32 v2, v1, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1730,25 +1722,24 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s6
+; GFX90A-NEXT: v_mov_b32_e32 v1, s6
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v1, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v0, v0
-; GFX90A-NEXT: v_min_f32_e32 v2, v1, v2
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1758,26 +1749,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v1, s6
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v0
-; GFX908-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1787,26 +1777,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mul_f32_e32 v3, 1.0, v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v1
; GFX8-NEXT: v_mov_b32_e32 v1, s6
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
-; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1833,32 +1822,31 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
-; GFX12-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], null offen
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], null offen
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s4, v0
-; GFX12-NEXT: v_readfirstlane_b32 s5, v1
+; GFX12-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-NEXT: v_readfirstlane_b32 s5, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
+; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[0:1], v[2:3]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1881,33 +1869,31 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
-; GFX11-NEXT: buffer_load_b64 v[0:1], v0, s[0:3], 0 offen
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v8, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s4, v0
-; GFX11-NEXT: v_readfirstlane_b32 s5, v1
+; GFX11-NEXT: v_readfirstlane_b32 s4, v2
+; GFX11-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v8, s4 :: v_dual_mov_b32 v9, s5
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen glc
+; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1939,33 +1925,30 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
-; GFX908-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s6, v0
-; GFX908-NEXT: v_readfirstlane_b32 s7, v1
-; GFX908-NEXT: v_mov_b32_e32 v9, s7
-; GFX908-NEXT: v_mov_b32_e32 v8, s6
+; GFX908-NEXT: v_readfirstlane_b32 s6, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: v_mov_b32_e32 v4, s6
+; GFX908-NEXT: v_mov_b32_e32 v5, s7
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1975,33 +1958,30 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
-; GFX8-NEXT: buffer_load_dwordx2 v[0:1], v0, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v8, s[16:19], 0 offen
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s6, v0
-; GFX8-NEXT: v_readfirstlane_b32 s7, v1
-; GFX8-NEXT: v_mov_b32_e32 v9, s7
-; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_readfirstlane_b32 s7, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v5, s7
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2028,31 +2008,30 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v2, s16
-; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen
+; GFX12-NEXT: v_mov_b32_e32 v6, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], null offen
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s4, v2
; GFX12-NEXT: v_readfirstlane_b32 s5, v3
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2075,32 +2054,30 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s16
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen
+; GFX11-NEXT: v_mov_b32_e32 v6, s16
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: buffer_load_b64 v[2:3], v6, s[0:3], 0 offen
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_readfirstlane_b32 s4, v2
; GFX11-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GFX11-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX11-NEXT: v_mov_b32_e32 v6, v2
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], 0 offen glc
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2132,31 +2109,30 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen
+; GFX908-NEXT: v_mov_b32_e32 v6, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s6, v2
; GFX908-NEXT: v_readfirstlane_b32 s7, v3
-; GFX908-NEXT: v_mov_b32_e32 v4, s6
-; GFX908-NEXT: v_mov_b32_e32 v5, s7
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
+; GFX908-NEXT: v_mov_b32_e32 v3, s7
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v9, v5
-; GFX908-NEXT: v_mov_b32_e32 v8, v4
-; GFX908-NEXT: v_mov_b32_e32 v7, v3
-; GFX908-NEXT: v_mov_b32_e32 v6, v2
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v6
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v7
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2166,31 +2142,30 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen
+; GFX8-NEXT: v_mov_b32_e32 v6, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v6, s[16:19], 0 offen
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s6, v2
; GFX8-NEXT: v_readfirstlane_b32 s7, v3
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v8, v4
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v6
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v7
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
index 8d9ed6596ea21..9552181b7d11b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
@@ -15,44 +15,42 @@ define amdgpu_cs void @memset_p1i8(ptr addrspace(1) %dst, i8 %val) {
; LOOP-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; LOOP-NEXT: v_or_b32_e32 v3, v4, v3
; LOOP-NEXT: s_mov_b64 s[0:1], 0
+; LOOP-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; LOOP-NEXT: v_bfe_u32 v5, v3, 8, 8
; LOOP-NEXT: s_mov_b32 s2, 0
; LOOP-NEXT: s_mov_b32 s3, 0xf000
+; LOOP-NEXT: v_lshrrev_b32_e32 v6, 24, v3
; LOOP-NEXT: .LBB0_1: ; %static-memset-expansion-main-body
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
-; LOOP-NEXT: s_waitcnt expcnt(1)
-; LOOP-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; LOOP-NEXT: v_bfe_u32 v5, v3, 8, 8
; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64
-; LOOP-NEXT: s_waitcnt expcnt(1)
-; LOOP-NEXT: v_lshrrev_b32_e32 v6, 24, v3
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:4
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:8
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:12
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:16
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:20
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:24
-; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:28
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:1
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:2
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:4
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:5
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:6
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:8
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:9
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:10
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:12
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:13
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:14
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:16
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:17
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:18
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:19
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:20
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:21
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:22
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:23
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:24
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:25
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:26
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:27
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:28
; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:29
; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:30
; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:31
@@ -62,6 +60,7 @@ define amdgpu_cs void @memset_p1i8(ptr addrspace(1) %dst, i8 %val) {
; LOOP-NEXT: s_cbranch_scc1 .LBB0_1
; LOOP-NEXT: ; %bb.2: ; %static-memset-post-expansion
; LOOP-NEXT: v_and_b32_e32 v2, 0xff, v2
+; LOOP-NEXT: s_waitcnt expcnt(3)
; LOOP-NEXT: v_lshlrev_b32_e32 v3, 8, v2
; LOOP-NEXT: v_or_b32_e32 v3, v2, v3
; LOOP-NEXT: s_waitcnt expcnt(1)
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
index f77a4940360e5..7bacd7a7e6811 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-flat-atomicrmw.ll
@@ -9402,14 +9402,14 @@ define void @flat_atomic_fmax_f32_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9433,13 +9433,13 @@ define void @flat_atomic_fmax_f32_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9466,16 +9466,16 @@ define void @flat_atomic_fmax_f32_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9494,17 +9494,16 @@ define void @flat_atomic_fmax_f32_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9533,14 +9532,14 @@ define void @flat_atomic_fmin_f32_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9564,13 +9563,13 @@ define void @flat_atomic_fmin_f32_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9597,16 +9596,16 @@ define void @flat_atomic_fmin_f32_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9625,17 +9624,16 @@ define void @flat_atomic_fmin_f32_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11563,14 +11561,14 @@ define void @flat_atomic_fmax_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB143_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11594,15 +11592,15 @@ define void @flat_atomic_fmax_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB143_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11629,16 +11627,16 @@ define void @flat_atomic_fmax_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB144_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11657,19 +11655,18 @@ define void @flat_atomic_fmax_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB144_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11698,14 +11695,14 @@ define void @flat_atomic_fmin_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB145_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11729,15 +11726,15 @@ define void @flat_atomic_fmin_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB145_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11764,16 +11761,16 @@ define void @flat_atomic_fmin_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11792,19 +11789,18 @@ define void @flat_atomic_fmin_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -11835,8 +11831,8 @@ define void @flat_atomic_fmaximum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -11902,8 +11898,8 @@ define void @flat_atomic_fmaximum_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
@@ -11973,8 +11969,8 @@ define void @flat_atomic_fminimum_v2f16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -12040,8 +12036,8 @@ define void @flat_atomic_fminimum_v2f16_ret_av_av(ptr %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
@@ -12113,30 +12109,30 @@ define void @flat_atomic_fadd_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12179,32 +12175,32 @@ define void @flat_atomic_fadd_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB152_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12246,30 +12242,30 @@ define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12293,18 +12289,18 @@ define void @flat_atomic_fsub_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12331,32 +12327,32 @@ define void @flat_atomic_fsub_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB154_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12375,22 +12371,21 @@ define void @flat_atomic_fsub_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB154_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12419,30 +12414,30 @@ define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12466,18 +12461,18 @@ define void @flat_atomic_fmax_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12504,32 +12499,32 @@ define void @flat_atomic_fmax_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB156_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12548,22 +12543,21 @@ define void @flat_atomic_fmax_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB156_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12592,30 +12586,30 @@ define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12639,18 +12633,18 @@ define void @flat_atomic_fmin_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12677,32 +12671,32 @@ define void @flat_atomic_fmin_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB158_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12721,22 +12715,21 @@ define void @flat_atomic_fmin_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB158_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12765,35 +12758,35 @@ define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12817,18 +12810,18 @@ define void @flat_atomic_fmaximum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_maximum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_maximum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12855,37 +12848,37 @@ define void @flat_atomic_fmaximum_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB160_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12904,22 +12897,21 @@ define void @flat_atomic_fmaximum_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB160_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_maximum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_maximum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12948,35 +12940,35 @@ define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -13000,18 +12992,18 @@ define void @flat_atomic_fminimum_v2bf16_ret_a_a(ptr %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_minimum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_minimum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -13038,37 +13030,37 @@ define void @flat_atomic_fminimum_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB162_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -13087,22 +13079,21 @@ define void @flat_atomic_fminimum_v2bf16_ret_av_av(ptr %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: flat_load_dword v3, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB162_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_minimum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_minimum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -13452,15 +13443,15 @@ define void @flat_atomic_nand_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB171_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_and_b32_e32 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_and_b32_e32 v0, v1, v4
; GFX90A-NEXT: v_not_b32_e32 v0, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -13484,14 +13475,14 @@ define void @flat_atomic_nand_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB171_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_bitop3_b32 v0, v1, v2, v1 bitop3:0x3f
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -13519,16 +13510,16 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_and_b32_e32 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_and_b32_e32 v0, v1, v4
; GFX90A-NEXT: v_not_b32_e32 v0, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -13548,16 +13539,16 @@ define void @flat_atomic_nand_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB172_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_bitop3_b32 v0, v1, v2, v1 bitop3:0x3f
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_bitop3_b32 v0, v1, v4, v1 bitop3:0x3f
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -14218,16 +14209,16 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB189_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -14251,16 +14242,16 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB189_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e32 v0, v1, v2
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -14288,17 +14279,17 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -14318,18 +14309,18 @@ define void @flat_atomic_usub_cond_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB190_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e32 v0, v1, v2
-; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_sub_u32_e32 v0, v1, v4
+; GFX950-NEXT: v_cmp_ge_u32_e32 vcc, v1, v4
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -14358,14 +14349,14 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB191_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -14389,14 +14380,14 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB191_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -14424,15 +14415,15 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -14452,16 +14443,16 @@ define void @flat_atomic_usub_sat_i32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB192_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_u32_e64 v0, v1, v2 clamp
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_u32_e64 v0, v1, v4 clamp
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -15303,15 +15294,14 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB201_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB201_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_and_b32_e32 v0, v3, v5
; GFX90A-NEXT: v_and_b32_e32 v8, v2, v4
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_not_b32_e32 v1, v0
; GFX90A-NEXT: v_not_b32_e32 v0, v8
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -15368,15 +15358,14 @@ define void @flat_atomic_nand_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB201_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB201_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_and_b32_e32 v0, v3, v5
; GFX950-NEXT: v_and_b32_e32 v8, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: v_not_b32_e32 v1, v0
; GFX950-NEXT: v_not_b32_e32 v0, v8
; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
@@ -15434,15 +15423,14 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB202_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB202_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_and_b32_e32 v0, v3, v5
; GFX90A-NEXT: v_and_b32_e32 v8, v2, v4
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_not_b32_e32 v1, v0
; GFX90A-NEXT: v_not_b32_e32 v0, v8
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -15493,15 +15481,14 @@ define void @flat_atomic_nand_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB202_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB202_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_and_b32_e32 v0, v3, v5
; GFX950-NEXT: v_and_b32_e32 v8, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: v_not_b32_e32 v1, v0
; GFX950-NEXT: v_not_b32_e32 v0, v8
; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
@@ -17232,8 +17219,8 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB219_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB219_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17241,7 +17228,6 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -17299,15 +17285,14 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB219_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB219_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
@@ -17371,8 +17356,8 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB220_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB220_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -17380,7 +17365,6 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX90A-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -17432,15 +17416,14 @@ define void @flat_atomic_usub_cond_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB220_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB220_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX950-NEXT: v_cmp_ge_u64_e32 vcc, v[2:3], v[4:5]
; GFX950-NEXT: s_nop 1
@@ -17502,15 +17485,14 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB221_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB221_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -17567,15 +17549,14 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB221_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB221_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
@@ -17637,15 +17618,14 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB222_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB222_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
; GFX90A-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
@@ -17696,15 +17676,14 @@ define void @flat_atomic_usub_sat_i64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB222_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .LBB222_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_sub_co_u32_e32 v0, vcc, v2, v4
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX950-NEXT: s_nop 1
; GFX950-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
@@ -17941,14 +17920,14 @@ define void @flat_atomic_fsub_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB225_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -17972,13 +17951,13 @@ define void @flat_atomic_fsub_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB225_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18006,15 +17985,15 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18034,15 +18013,15 @@ define void @flat_atomic_fsub_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .LBB226_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_sub_f32_e32 v0, v1, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_sub_f32_e32 v0, v1, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18071,16 +18050,16 @@ define void @flat_atomic_fmax_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB227_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18104,15 +18083,15 @@ define void @flat_atomic_fmax_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB227_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18139,18 +18118,18 @@ define void @flat_atomic_fmax_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB228_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18169,19 +18148,18 @@ define void @flat_atomic_fmax_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB228_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18210,16 +18188,16 @@ define void @flat_atomic_fmin_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB229_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18243,15 +18221,15 @@ define void @flat_atomic_fmin_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB229_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18278,18 +18256,18 @@ define void @flat_atomic_fmin_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB230_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18308,19 +18286,18 @@ define void @flat_atomic_fmin_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .LBB230_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18349,17 +18326,17 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB231_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18383,14 +18360,14 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB231_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_maximum3_f32 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18418,18 +18395,18 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB232_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18449,16 +18426,16 @@ define void @flat_atomic_fmaximum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB232_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_maximum3_f32 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_maximum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18487,17 +18464,17 @@ define void @flat_atomic_fminimum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB233_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_min_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_min_f32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18521,14 +18498,14 @@ define void @flat_atomic_fminimum_f32_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB233_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_minimum3_f32 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18556,18 +18533,18 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB234_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_min_f32_e32 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_min_f32_e32 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -18587,16 +18564,16 @@ define void @flat_atomic_fminimum_f32_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB234_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_minimum3_f32 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_minimum3_f32 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -18929,14 +18906,13 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB237_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB237_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -18987,15 +18963,14 @@ define void @flat_atomic_fsub_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB237_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB237_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -19048,14 +19023,13 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB238_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB238_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -19100,15 +19074,14 @@ define void @flat_atomic_fsub_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB238_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB238_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_add_f64 v[0:1], v[2:3], -v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -19571,19 +19544,18 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB243_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX90A-NEXT: .LBB243_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]
; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] glc
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -19637,21 +19609,20 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB243_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB243_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] sc0
+; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -19707,19 +19678,18 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB244_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX90A-NEXT: .LBB244_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]
; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] glc
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -19767,21 +19737,20 @@ define void @flat_atomic_fmaximum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB244_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB244_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_max_f64 v[0:1], v[2:3], v[4:5]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] sc0
+; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -19836,19 +19805,18 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_cbranch_scc0 .LBB245_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX90A-NEXT: .LBB245_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]
; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] glc
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -19902,21 +19870,20 @@ define void @flat_atomic_fminimum_f64_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
; GFX950-NEXT: s_cbranch_scc0 .LBB245_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB245_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] sc0
+; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -19972,19 +19939,18 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_cbranch_scc0 .LBB246_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX90A-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX90A-NEXT: .LBB246_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]
; GFX90A-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX90A-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] glc
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20032,21 +19998,20 @@ define void @flat_atomic_fminimum_f64_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_cbranch_scc0 .LBB246_4
; GFX950-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX950-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX950-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_mov_b32_e32 v6, 0x7ff80000
+; GFX950-NEXT: v_mov_b32_e32 v8, 0x7ff80000
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB246_2: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_min_f64 v[0:1], v[2:3], v[4:5]
; GFX950-NEXT: v_cmp_u_f64_e32 vcc, v[2:3], v[4:5]
-; GFX950-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
+; GFX950-NEXT: s_nop 1
+; GFX950-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc
; GFX950-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[8:9], v[0:3] sc0
+; GFX950-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20097,14 +20062,14 @@ define void @flat_atomic_fadd_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB247_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20150,15 +20115,15 @@ define void @flat_atomic_fadd_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB248_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v2
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20202,14 +20167,14 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB249_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20233,14 +20198,14 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB249_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_add_f16 v0, v1, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20268,15 +20233,15 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_add_f16 v0, v1, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20296,16 +20261,16 @@ define void @flat_atomic_fsub_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB250_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_add_f16 v0, v1, v2 neg_lo:[0,1] neg_hi:[0,1]
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_add_f16 v0, v1, v4 neg_lo:[0,1] neg_hi:[0,1]
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20334,16 +20299,16 @@ define void @flat_atomic_fmax_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB251_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20367,16 +20332,17 @@ define void @flat_atomic_fmax_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB251_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20403,18 +20369,18 @@ define void @flat_atomic_fmax_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB252_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20433,20 +20399,20 @@ define void @flat_atomic_fmax_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB252_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20475,16 +20441,16 @@ define void @flat_atomic_fmin_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB253_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_min_f16 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20508,16 +20474,17 @@ define void @flat_atomic_fmin_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB253_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_min_f16 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20544,18 +20511,18 @@ define void @flat_atomic_fmin_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB254_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_min_f16 v0, v3, v0
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
@@ -20574,20 +20541,20 @@ define void @flat_atomic_fmin_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB254_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v2, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_min_f16 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20616,21 +20583,21 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v3, v0, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20654,14 +20621,14 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20689,22 +20656,22 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB256_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v3, v0, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20724,16 +20691,16 @@ define void @flat_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB256_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_maximum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20762,21 +20729,21 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_min_f16 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v3, v0, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_min_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20800,14 +20767,14 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20835,22 +20802,22 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7e00
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB258_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_min_f16 v0, v1, v2
-; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v2 src0_sel:WORD_1 src1_sel:WORD_1
-; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v6, v3, v0, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_min_f16 v0, v1, v4
+; GFX90A-NEXT: v_cmp_o_f16_sdwa vcc, v1, v4 src0_sel:WORD_1 src1_sel:WORD_1
+; GFX90A-NEXT: v_cmp_o_f16_e64 s[4:5], v1, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v6, v5, v0, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX90A-NEXT: v_perm_b32 v0, v0, v6, s8
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20870,16 +20837,16 @@ define void @flat_atomic_fminimum_v2f16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v4
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB258_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v2, v2
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_pk_minimum3_f16 v0, v1, v4, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -20912,32 +20879,32 @@ define void @flat_atomic_fadd_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_add_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -20982,34 +20949,34 @@ define void @flat_atomic_fadd_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB260_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_add_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21053,32 +21020,32 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21102,20 +21069,20 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21142,34 +21109,34 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB262_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21188,24 +21155,23 @@ define void @flat_atomic_fsub_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB262_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_sub_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21234,32 +21200,32 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21283,20 +21249,20 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21323,34 +21289,34 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB264_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21369,24 +21335,23 @@ define void @flat_atomic_fmax_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB264_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21415,32 +21380,32 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21464,20 +21429,20 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21504,34 +21469,34 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB266_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21550,24 +21515,23 @@ define void @flat_atomic_fmin_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB266_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX950-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21596,37 +21560,37 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v0
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v3, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21650,20 +21614,20 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_maximum3_f32 v0, v3, v0, v0
-; GFX950-NEXT: v_maximum3_f32 v3, v7, v6, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_maximum3_f32 v0, v0, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21690,39 +21654,39 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB268_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v0
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v3, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21741,24 +21705,23 @@ define void @flat_atomic_fmaximum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB268_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_maximum3_f32 v0, v3, v0, v0
-; GFX950-NEXT: v_maximum3_f32 v3, v7, v6, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_maximum3_f32 v0, v0, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21787,37 +21750,37 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v0
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v3, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21841,20 +21804,20 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr inreg %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_minimum3_f32 v0, v3, v0, v0
-; GFX950-NEXT: v_minimum3_f32 v3, v7, v6, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_minimum3_f32 v0, v0, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
@@ -21881,39 +21844,39 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dword v1, v[0:1] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v3, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v2
-; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[16:17], s[16:17] op_sel:[0,1]
; GFX90A-NEXT: .LBB270_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v0
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v3, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v8, v0, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v0, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[16:17], s[16:17] op_sel:[0,1]
-; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 glc
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v7, v0, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
@@ -21932,24 +21895,23 @@ define void @flat_atomic_fminimum_v2bf16_saddr_ret_av_av(ptr inreg %ptr) #0 {
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX950-NEXT: flat_load_dword v1, v[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v2
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB270_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX950-NEXT: v_minimum3_f32 v0, v3, v0, v0
-; GFX950-NEXT: v_minimum3_f32 v3, v7, v6, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v3, v0
-; GFX950-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] offset:40 sc0
+; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_minimum3_f32 v0, v0, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
diff --git a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
index 9458b1cbd28c4..f31448a0ddc1c 100644
--- a/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
+++ b/llvm/test/CodeGen/AMDGPU/a-v-global-atomicrmw.ll
@@ -6247,14 +6247,14 @@ define void @global_atomic_fmax_f32_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6278,13 +6278,13 @@ define void @global_atomic_fmax_f32_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6311,16 +6311,16 @@ define void @global_atomic_fmax_f32_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6339,17 +6339,16 @@ define void @global_atomic_fmax_f32_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6378,14 +6377,14 @@ define void @global_atomic_fmin_f32_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6409,13 +6408,13 @@ define void @global_atomic_fmin_f32_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6442,16 +6441,16 @@ define void @global_atomic_fmin_f32_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -6470,17 +6469,16 @@ define void @global_atomic_fmin_f32_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v2, v2
; GFX950-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v5, v3, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
+; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7617,14 +7615,14 @@ define void @global_atomic_fmax_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB143_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7648,15 +7646,15 @@ define void @global_atomic_fmax_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB143_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7683,16 +7681,16 @@ define void @global_atomic_fmax_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB144_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7711,19 +7709,18 @@ define void @global_atomic_fmax_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB144_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_max_f16 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7752,14 +7749,14 @@ define void @global_atomic_fmin_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB145_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7783,15 +7780,15 @@ define void @global_atomic_fmin_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB145_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7818,16 +7815,16 @@ define void @global_atomic_fmin_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
+; GFX90A-NEXT: ; def v2
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v5, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7846,19 +7843,18 @@ define void @global_atomic_fmin_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v2, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v2, v4, v4
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v5, v3, v3
+; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v2, v5, v2
+; GFX950-NEXT: v_pk_min_f16 v2, v2, v4
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -7889,8 +7885,8 @@ define void @global_atomic_fmaximum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7956,8 +7952,8 @@ define void @global_atomic_fmaximum_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
@@ -8027,8 +8023,8 @@ define void @global_atomic_fminimum_v2f16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8094,8 +8090,8 @@ define void @global_atomic_fminimum_v2f16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v4
; GFX90A-NEXT: ;;#ASMEND
@@ -8167,30 +8163,30 @@ define void @global_atomic_fadd_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8233,32 +8229,32 @@ define void @global_atomic_fadd_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB152_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8300,30 +8296,30 @@ define void @global_atomic_fsub_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8347,18 +8343,18 @@ define void @global_atomic_fsub_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB153_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8385,32 +8381,32 @@ define void @global_atomic_fsub_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB154_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8429,22 +8425,21 @@ define void @global_atomic_fsub_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB154_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8473,30 +8468,30 @@ define void @global_atomic_fmax_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8520,18 +8515,18 @@ define void @global_atomic_fmax_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB155_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8558,32 +8553,32 @@ define void @global_atomic_fmax_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB156_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8602,22 +8597,21 @@ define void @global_atomic_fmax_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB156_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8646,30 +8640,30 @@ define void @global_atomic_fmin_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8693,18 +8687,18 @@ define void @global_atomic_fmin_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB157_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8731,32 +8725,32 @@ define void @global_atomic_fmin_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB158_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX90A-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v2, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v5, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8775,22 +8769,21 @@ define void @global_atomic_fmin_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB158_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX950-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8819,35 +8812,35 @@ define void @global_atomic_fmaximum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8871,18 +8864,18 @@ define void @global_atomic_fmaximum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB159_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_maximum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_maximum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8909,37 +8902,37 @@ define void @global_atomic_fmaximum_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB160_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_max_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -8958,22 +8951,21 @@ define void @global_atomic_fmaximum_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB160_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_maximum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_maximum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_maximum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_maximum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9002,35 +8994,35 @@ define void @global_atomic_fminimum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v2, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9054,18 +9046,18 @@ define void @global_atomic_fminimum_v2bf16_ret_a_a(ptr addrspace(1) %ptr) #0 {
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[0:1], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v4, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v2, a0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB161_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_minimum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_minimum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9092,37 +9084,37 @@ define void @global_atomic_fminimum_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v2
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v5, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v4
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB162_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v9, v6, v2
-; GFX90A-NEXT: v_min_f32_e32 v10, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v8, v7
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v6, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: s_waitcnt vmcnt(0)
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v2, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v5, v9, vcc
+; GFX90A-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX90A-NEXT: v_add3_u32 v8, v8, v2, s8
+; GFX90A-NEXT: v_add3_u32 v10, v10, v7, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v8, v9, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v7, v2, s9
; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -9141,22 +9133,21 @@ define void @global_atomic_fminimum_v2bf16_ret_av_av(ptr addrspace(1) %ptr) #0 {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: global_load_dword v3, v[0:1], off offset:40
-; GFX950-NEXT: s_mov_b64 s[0:1], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v4
+; GFX950-NEXT: ; def v2
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[0:1], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB162_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_minimum3_f32 v2, v5, v2, v2
-; GFX950-NEXT: v_minimum3_f32 v5, v7, v6, v6
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v5, v2
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX950-NEXT: v_minimum3_f32 v2, v2, v4, v4
+; GFX950-NEXT: v_minimum3_f32 v6, v6, v5, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v6, v2
; GFX950-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -12079,14 +12070,14 @@ define void @global_atomic_fmax_f32_saddr_ret_a_a(ptr addrspace(1) inreg %ptr) #
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: .LBB227_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12111,13 +12102,13 @@ define void @global_atomic_fmax_f32_saddr_ret_a_a(ptr addrspace(1) inreg %ptr) #
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_max_f32_e32 v3, v0, v0
; GFX950-NEXT: .LBB227_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12145,16 +12136,16 @@ define void @global_atomic_fmax_f32_saddr_ret_av_av(ptr addrspace(1) inreg %ptr)
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: .LBB228_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12174,17 +12165,16 @@ define void @global_atomic_fmax_f32_saddr_ret_av_av(ptr addrspace(1) inreg %ptr)
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v3, v0, v0
; GFX950-NEXT: .LBB228_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12214,14 +12204,14 @@ define void @global_atomic_fmin_f32_saddr_ret_a_a(ptr addrspace(1) inreg %ptr) #
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: .LBB229_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12246,13 +12236,13 @@ define void @global_atomic_fmin_f32_saddr_ret_a_a(ptr addrspace(1) inreg %ptr) #
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_max_f32_e32 v3, v0, v0
; GFX950-NEXT: .LBB229_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12280,16 +12270,16 @@ define void @global_atomic_fmin_f32_saddr_ret_av_av(ptr addrspace(1) inreg %ptr)
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v3, v0, v0
; GFX90A-NEXT: .LBB230_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -12309,17 +12299,16 @@ define void @global_atomic_fmin_f32_saddr_ret_av_av(ptr addrspace(1) inreg %ptr)
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v3, v0, v0
; GFX950-NEXT: .LBB230_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_max_f32_e32 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13497,14 +13486,14 @@ define void @global_atomic_fmax_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v3, v0, v0
; GFX90A-NEXT: .LBB251_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v1, v1
-; GFX90A-NEXT: v_pk_max_f16 v0, v4, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13529,15 +13518,15 @@ define void @global_atomic_fmax_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr)
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_pk_max_f16 v3, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB251_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v4, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v4, v0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13565,16 +13554,16 @@ define void @global_atomic_fmax_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg %pt
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v3, v0, v0
; GFX90A-NEXT: .LBB252_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v1, v1
-; GFX90A-NEXT: v_pk_max_f16 v0, v4, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13594,19 +13583,18 @@ define void @global_atomic_fmax_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg %pt
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v3, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB252_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v4, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v4, v0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13636,14 +13624,14 @@ define void @global_atomic_fmin_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr)
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v3, v0, v0
; GFX90A-NEXT: .LBB253_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v1, v1
-; GFX90A-NEXT: v_pk_min_f16 v0, v4, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13668,15 +13656,15 @@ define void @global_atomic_fmin_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr)
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_pk_max_f16 v3, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB253_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v4, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v4, v0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13704,16 +13692,16 @@ define void @global_atomic_fmin_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg %pt
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
-; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
+; GFX90A-NEXT: ; def v0
; GFX90A-NEXT: ;;#ASMEND
+; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v3, v0, v0
; GFX90A-NEXT: .LBB254_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v1, v1
-; GFX90A-NEXT: v_pk_min_f16 v0, v4, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v3
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13733,19 +13721,18 @@ define void @global_atomic_fmin_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg %pt
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v3, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB254_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_max_f16 v0, v3, v3
; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v4, v1, v1
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_pk_min_f16 v0, v4, v0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v3
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -13777,8 +13764,8 @@ define void @global_atomic_fmaximum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB255_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -13846,8 +13833,8 @@ define void @global_atomic_fmaximum_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v3
; GFX90A-NEXT: ;;#ASMEND
@@ -13919,8 +13906,8 @@ define void @global_atomic_fminimum_v2f16_saddr_ret_a_a(ptr addrspace(1) inreg %
; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: .LBB257_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -13988,8 +13975,8 @@ define void @global_atomic_fminimum_v2f16_saddr_ret_av_av(ptr addrspace(1) inreg
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
-; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7e00
+; GFX90A-NEXT: s_mov_b32 s8, 0x5040100
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def v3
; GFX90A-NEXT: ;;#ASMEND
@@ -14063,30 +14050,30 @@ define void @global_atomic_fadd_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB259_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14131,32 +14118,32 @@ define void @global_atomic_fadd_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB260_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14200,30 +14187,30 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14248,18 +14235,18 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB261_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14287,32 +14274,32 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB262_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14332,22 +14319,21 @@ define void @global_atomic_fsub_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB262_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_sub_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14377,30 +14363,30 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14425,18 +14411,18 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB263_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14464,32 +14450,32 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB264_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14509,22 +14495,21 @@ define void @global_atomic_fmax_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB264_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_max_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14554,30 +14539,30 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14602,18 +14587,18 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg %ptr
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB265_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14641,32 +14626,32 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB266_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v4, v0
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v4, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14686,22 +14671,21 @@ define void @global_atomic_fmin_v2bf16_saddr_ret_av_av(ptr addrspace(1) inreg %p
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB266_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_min_f32_e32 v0, v4, v0
-; GFX950-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX950-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14731,35 +14715,35 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v8, v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v5, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14784,18 +14768,18 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB267_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_maximum3_f32 v0, v4, v0, v0
-; GFX950-NEXT: v_maximum3_f32 v4, v6, v5, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_maximum3_f32 v0, v0, v3, v3
+; GFX950-NEXT: v_maximum3_f32 v5, v5, v4, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14823,37 +14807,37 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_av_av(ptr addrspace(1) inre
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB268_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v8, v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v9, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v5, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14873,22 +14857,21 @@ define void @global_atomic_fmaximum_v2bf16_saddr_ret_av_av(ptr addrspace(1) inre
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB268_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_maximum3_f32 v0, v4, v0, v0
-; GFX950-NEXT: v_maximum3_f32 v4, v6, v5, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_maximum3_f32 v0, v0, v3, v3
+; GFX950-NEXT: v_maximum3_f32 v5, v5, v4, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14918,35 +14901,35 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX90A-NEXT: ;;#ASMSTART
; GFX90A-NEXT: ; def a0
; GFX90A-NEXT: ;;#ASMEND
-; GFX90A-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX90A-NEXT: v_accvgpr_read_b32 v0, a0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v8, v5, v0
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v5, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -14971,18 +14954,18 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_a_a(ptr addrspace(1) inreg
; GFX950-NEXT: ; def a0
; GFX950-NEXT: ;;#ASMEND
; GFX950-NEXT: s_mov_b64 s[2:3], 0
-; GFX950-NEXT: v_accvgpr_read_b32 v3, a0
+; GFX950-NEXT: v_accvgpr_read_b32 v0, a0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB269_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_minimum3_f32 v0, v4, v0, v0
-; GFX950-NEXT: v_minimum3_f32 v4, v6, v5, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_minimum3_f32 v0, v0, v3, v3
+; GFX950-NEXT: v_minimum3_f32 v5, v5, v4, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -15010,37 +14993,37 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_av_av(ptr addrspace(1) inre
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: global_load_dword v1, v2, s[16:17] offset:40
+; GFX90A-NEXT: ;;#ASMSTART
+; GFX90A-NEXT: ; def v0
+; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
-; GFX90A-NEXT: v_mov_b32_e32 v4, 0x7fc00000
-; GFX90A-NEXT: ;;#ASMSTART
-; GFX90A-NEXT: ; def v3
-; GFX90A-NEXT: ;;#ASMEND
; GFX90A-NEXT: .LBB270_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v8, v5, v0
-; GFX90A-NEXT: v_min_f32_e32 v9, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v7, v6
-; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v5, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v8, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v4, v9, vcc
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v7, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e32 vcc, v6, v5
+; GFX90A-NEXT: v_cmp_o_f32_e64 s[4:5], v0, v3
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v4, v8, vcc
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[16:17] offset:40 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -15060,22 +15043,21 @@ define void @global_atomic_fminimum_v2bf16_saddr_ret_av_av(ptr addrspace(1) inre
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b32_e32 v2, 0
; GFX950-NEXT: global_load_dword v1, v2, s[0:1] offset:40
-; GFX950-NEXT: s_mov_b64 s[2:3], 0
; GFX950-NEXT: ;;#ASMSTART
-; GFX950-NEXT: ; def v3
+; GFX950-NEXT: ; def v0
; GFX950-NEXT: ;;#ASMEND
+; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB270_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: s_nop 0
-; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
-; GFX950-NEXT: s_waitcnt vmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX950-NEXT: v_minimum3_f32 v0, v4, v0, v0
-; GFX950-NEXT: v_minimum3_f32 v4, v6, v5, v5
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v4, v0
+; GFX950-NEXT: s_waitcnt vmcnt(0)
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX950-NEXT: v_minimum3_f32 v0, v0, v3, v3
+; GFX950-NEXT: v_minimum3_f32 v5, v5, v4, v4
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v5, v0
; GFX950-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0)
; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 0fa5e932e7cea..16bc383b1999d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -2730,17 +2730,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-LABEL: add_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
@@ -2783,17 +2785,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_ITERATIVE-LABEL: add_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX8_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
+; GFX8_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_1
@@ -2833,17 +2837,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_ITERATIVE-LABEL: add_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s6, m0
-; GFX9_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
+; GFX9_ITERATIVE-NEXT: s_add_u32 s6, s6, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s7, m0
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_1
@@ -2882,6 +2888,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1064_ITERATIVE-LABEL: add_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -2889,10 +2896,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s2
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s2
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s2
; GFX1064_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s7, s7, s8
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_1
@@ -2932,6 +2940,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_ITERATIVE-LABEL: add_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -2939,10 +2948,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s0
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s1
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s7, s1
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s6, s1
; GFX1032_ITERATIVE-NEXT: s_add_u32 s6, s6, s2
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_1
@@ -2981,7 +2991,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1164_ITERATIVE-LABEL: add_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -2989,11 +3000,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v3, s2
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v2, s2
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s2
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s2
; GFX1164_ITERATIVE-NEXT: s_add_u32 s6, s6, s3
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s7, s7, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
@@ -3036,7 +3048,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_ITERATIVE-LABEL: add_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -3044,11 +3056,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s1
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s1
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s1
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s1
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1
; GFX1132_ITERATIVE-NEXT: s_add_u32 s6, s6, s2
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s7, s7, 0
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s7, s7, s3
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1
@@ -3088,16 +3101,17 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1264_ITERATIVE-LABEL: add_i64_varying:
; GFX1264_ITERATIVE: ; %bb.0: ; %entry
-; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1264_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[0:1]
; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s8
+; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s8
+; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s8
; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s8
; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s8
; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
@@ -3142,23 +3156,23 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_ITERATIVE-LABEL: add_i64_varying:
; GFX1232_ITERATIVE: ; %bb.0: ; %entry
-; GFX1232_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1232_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1232_ITERATIVE-NEXT: s_mov_b32 s1, 0
; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1232_ITERATIVE-NEXT: .LBB5_1: ; %ComputeLoop
; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s0, v2, s3
-; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s3
-; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s3
-; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s1
+; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s1
+; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s1
+; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1
+; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
-; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[0:1]
+; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1
+; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]
; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB5_1
; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
@@ -3184,7 +3198,6 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x0
; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v2
; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_ITERATIVE-NEXT: v_add_co_u32 v0, vcc_lo, s2, v0
; GFX1232_ITERATIVE-NEXT: v_add_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
@@ -4671,7 +4684,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-LABEL: sub_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v2, s7, v0
@@ -4681,20 +4694,20 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_cbranch_execz .LBB7_4
; GFX7LESS-NEXT: ; %bb.1:
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_load_dword s12, s[4:5], 0x0
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
-; GFX7LESS-NEXT: s_mul_i32 s11, s10, s6
+; GFX7LESS-NEXT: s_load_dword s4, s[2:3], 0x0
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s5, s[6:7]
+; GFX7LESS-NEXT: s_mov_b64 s[10:11], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mul_i32 s13, s12, s5
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s12
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v4, v0
-; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s11, v4
+; GFX7LESS-NEXT: v_subrev_i32_e32 v3, vcc, s13, v4
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v0, v3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, v4
@@ -4702,15 +4715,15 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[10:11], vcc, s[10:11]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: .LBB7_4: ; %Flow3
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v1, s10, v2
+; GFX7LESS-NEXT: v_mul_lo_u32 v1, s12, v2
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v0
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
@@ -7018,23 +7031,23 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_and_saveexec_b64 s[10:11], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB10_4
; GFX7LESS-NEXT: ; %bb.1:
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_mov_b32 s5, s3
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s2
-; GFX7LESS-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX7LESS-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX7LESS-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX7LESS-NEXT: s_mul_i32 s3, s9, s2
-; GFX7LESS-NEXT: s_mul_i32 s2, s8, s2
-; GFX7LESS-NEXT: v_mov_b32_e32 v6, s2
-; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s3, v0
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], 0
+; GFX7LESS-NEXT: s_mul_i32 s7, s9, s6
+; GFX7LESS-NEXT: s_mul_i32 s6, s8, s6
+; GFX7LESS-NEXT: v_mov_b32_e32 v6, s6
+; GFX7LESS-NEXT: v_add_i32_e32 v5, vcc, s7, v0
+; GFX7LESS-NEXT: s_mov_b64 s[12:13], 0
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s6
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s7
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
+; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_mov_b32_e32 v9, v0
@@ -7050,11 +7063,11 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS-NEXT: buffer_wbinvl1
; GFX7LESS-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
-; GFX7LESS-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
+; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB10_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[12:13]
; GFX7LESS-NEXT: .LBB10_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
@@ -7151,31 +7164,31 @@ define amdgpu_kernel void @sub_i64_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9-NEXT: s_mul_i32 s7, s9, s6
; GFX9-NEXT: s_mul_hi_u32 s12, s8, s6
-; GFX9-NEXT: s_add_i32 s14, s12, s7
-; GFX9-NEXT: s_mul_i32 s15, s8, s6
+; GFX9-NEXT: s_add_i32 s7, s12, s7
+; GFX9-NEXT: s_mul_i32 s14, s8, s6
; GFX9-NEXT: s_mov_b64 s[12:13], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: .LBB10_2: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v7, v0
-; GFX9-NEXT: v_mov_b32_e32 v8, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s14
-; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s15, v7
-; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v0, v5
-; GFX9-NEXT: v_mov_b32_e32 v1, v6
-; GFX9-NEXT: v_mov_b32_e32 v2, v7
-; GFX9-NEXT: v_mov_b32_e32 v3, v8
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s14, v8
+; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v5, vcc
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-NEXT: v_mov_b32_e32 v2, v8
+; GFX9-NEXT: v_mov_b32_e32 v3, v9
; GFX9-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9-NEXT: s_cbranch_execnz .LBB10_2
@@ -7635,17 +7648,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-LABEL: sub_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7LESS_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
@@ -7662,31 +7677,31 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX7LESS_ITERATIVE-NEXT: ; %bb.3:
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX7LESS_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s6, -1
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s14
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s15
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s5, s3
; GFX7LESS_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v7, vcc, s8, v9
+; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, s9
-; GFX7LESS_ITERATIVE-NEXT: v_subrev_i32_e32 v6, vcc, s8, v8
-; GFX7LESS_ITERATIVE-NEXT: v_subb_u32_e32 v7, vcc, v9, v0, vcc
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
; GFX7LESS_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX7LESS_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX7LESS_ITERATIVE-NEXT: buffer_wbinvl1
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX7LESS_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX7LESS_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
@@ -7709,17 +7724,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_ITERATIVE-LABEL: sub_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
-; GFX8_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
+; GFX8_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
@@ -7734,30 +7751,30 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX8_ITERATIVE-NEXT: ; %bb.3:
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_ITERATIVE-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX8_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
; GFX8_ITERATIVE-NEXT: s_mov_b32 s6, -1
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX8_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s14
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s15
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX8_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX8_ITERATIVE-NEXT: s_mov_b32 s5, s3
; GFX8_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, s9
-; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v6, vcc, s8, v8
-; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v7, vcc, v9, v0, vcc
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX8_ITERATIVE-NEXT: v_subrev_u32_e32 v7, vcc, s8, v9
+; GFX8_ITERATIVE-NEXT: v_subb_u32_e32 v8, vcc, v10, v6, vcc
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
; GFX8_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX8_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX8_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX8_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX8_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX8_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX8_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
@@ -7779,17 +7796,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_ITERATIVE-LABEL: sub_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX9_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s2
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s2
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s2
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v4, s8, m0
-; GFX9_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
+; GFX9_ITERATIVE-NEXT: s_add_u32 s8, s8, s6
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v5, s9, m0
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
@@ -7804,30 +7823,30 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_ITERATIVE-NEXT: s_cbranch_execz .LBB11_6
; GFX9_ITERATIVE-NEXT: ; %bb.3:
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_ITERATIVE-NEXT: s_load_dwordx2 s[14:15], s[2:3], 0x0
+; GFX9_ITERATIVE-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[12:13], 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v6, s9
; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0xf000
; GFX9_ITERATIVE-NEXT: s_mov_b32 s6, -1
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX9_ITERATIVE-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s14
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s15
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s4
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, s5
+; GFX9_ITERATIVE-NEXT: s_mov_b32 s4, s2
; GFX9_ITERATIVE-NEXT: s_mov_b32 s5, s3
; GFX9_ITERATIVE-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v8, v0
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v9, v1
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, s9
-; GFX9_ITERATIVE-NEXT: v_subrev_co_u32_e32 v6, vcc, s8, v8
-; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v7, vcc, v9, v0, vcc
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v6
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v7
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, v8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, v9
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v9, v0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v10, v1
+; GFX9_ITERATIVE-NEXT: v_subrev_co_u32_e32 v7, vcc, s8, v9
+; GFX9_ITERATIVE-NEXT: v_subb_co_u32_e32 v8, vcc, v10, v6, vcc
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v0, v7
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, v8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v2, v9
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, v10
; GFX9_ITERATIVE-NEXT: buffer_atomic_cmpswap_x2 v[0:3], off, s[4:7], 0 glc
; GFX9_ITERATIVE-NEXT: s_waitcnt vmcnt(0)
; GFX9_ITERATIVE-NEXT: buffer_wbinvl1_vol
-; GFX9_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX9_ITERATIVE-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[9:10]
; GFX9_ITERATIVE-NEXT: s_or_b64 s[12:13], vcc, s[12:13]
; GFX9_ITERATIVE-NEXT: s_andn2_b64 exec, exec, s[12:13]
; GFX9_ITERATIVE-NEXT: s_cbranch_execnz .LBB11_4
@@ -7848,6 +7867,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1064_ITERATIVE-LABEL: sub_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
@@ -7855,10 +7875,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s2, s[0:1]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s2
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s2
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s2
; GFX1064_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s9, s9, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
@@ -7916,6 +7937,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1032_ITERATIVE-LABEL: sub_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
@@ -7923,10 +7945,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s1, s0
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s1
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s1
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s1
; GFX1032_ITERATIVE-NEXT: s_add_u32 s8, s8, s2
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s0, s0, s1
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
@@ -7983,6 +8006,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1164_ITERATIVE-LABEL: sub_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
@@ -7992,10 +8016,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s2, s[0:1]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s3, v0, s2
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s2
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s2
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s2
; GFX1164_ITERATIVE-NEXT: s_add_u32 s8, s8, s3
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s9, s9, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[2:3], 1, s2
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
@@ -8062,7 +8087,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1132_ITERATIVE-LABEL: sub_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[8:9], 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr4_vgpr5
@@ -8071,10 +8096,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s1
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s3, v1, s1
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v5, s9, s1
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v4, s8, s1
; GFX1132_ITERATIVE-NEXT: s_add_u32 s8, s8, s2
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s9, s9, 0
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s9, s9, s3
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1
@@ -8134,16 +8160,17 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1264_ITERATIVE-LABEL: sub_i64_varying:
; GFX1264_ITERATIVE: ; %bb.0: ; %entry
-; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1264_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1264_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[0:1], exec
; GFX1264_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1264_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1264_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1264_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1264_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1264_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[0:1]
; GFX1264_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s8
+; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s8
+; GFX1264_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s8
; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s8
; GFX1264_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s8
; GFX1264_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
@@ -8188,23 +8215,23 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
;
; GFX1232_ITERATIVE-LABEL: sub_i64_varying:
; GFX1232_ITERATIVE: ; %bb.0: ; %entry
-; GFX1232_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1232_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
+; GFX1232_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1232_ITERATIVE-NEXT: s_mov_b32 s0, exec_lo
; GFX1232_ITERATIVE-NEXT: s_mov_b64 s[6:7], 0
-; GFX1232_ITERATIVE-NEXT: s_mov_b32 s1, 0
; GFX1232_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1232_ITERATIVE-NEXT: .LBB11_1: ; %ComputeLoop
; GFX1232_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1232_ITERATIVE-NEXT: s_ctz_i32_b32 s1, s0
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s0, v2, s3
-; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s3
-; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s3
-; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s3, v2, s1
+; GFX1232_ITERATIVE-NEXT: v_readlane_b32 s2, v3, s1
+; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v1, s7, s1
+; GFX1232_ITERATIVE-NEXT: v_writelane_b32 v0, s6, s1
+; GFX1232_ITERATIVE-NEXT: s_lshl_b32 s1, 1, s1
; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
-; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[0:1]
+; GFX1232_ITERATIVE-NEXT: s_and_not1_b32 s0, s0, s1
+; GFX1232_ITERATIVE-NEXT: s_add_nc_u64 s[6:7], s[6:7], s[2:3]
; GFX1232_ITERATIVE-NEXT: s_cbranch_scc1 .LBB11_1
; GFX1232_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1232_ITERATIVE-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
@@ -8230,7 +8257,6 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX1232_ITERATIVE-NEXT: s_wait_kmcnt 0x0
; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s2, v2
; GFX1232_ITERATIVE-NEXT: v_readfirstlane_b32 s3, v3
-; GFX1232_ITERATIVE-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1232_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1232_ITERATIVE-NEXT: v_sub_co_u32 v0, vcc_lo, s2, v0
; GFX1232_ITERATIVE-NEXT: v_sub_co_ci_u32_e64 v1, null, s3, v1, vcc_lo
@@ -8345,11 +8371,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX8_DPP-NEXT: v_mov_b32_e32 v6, s14
; GFX8_DPP-NEXT: v_mov_b32_e32 v7, s15
; GFX8_DPP-NEXT: s_mov_b32 s5, s3
+; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s11
; GFX8_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX8_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX8_DPP-NEXT: v_mov_b32_e32 v11, v7
-; GFX8_DPP-NEXT: v_mov_b32_e32 v0, s11
; GFX8_DPP-NEXT: v_subrev_u32_e32 v8, vcc, s10, v10
; GFX8_DPP-NEXT: v_subb_u32_e32 v9, vcc, v11, v0, vcc
; GFX8_DPP-NEXT: v_mov_b32_e32 v6, v8
@@ -8448,11 +8474,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out, ptr addrspace(
; GFX9_DPP-NEXT: v_mov_b32_e32 v6, s14
; GFX9_DPP-NEXT: v_mov_b32_e32 v7, s15
; GFX9_DPP-NEXT: s_mov_b32 s5, s3
+; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s11
; GFX9_DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX9_DPP-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_DPP-NEXT: v_mov_b32_e32 v10, v6
; GFX9_DPP-NEXT: v_mov_b32_e32 v11, v7
-; GFX9_DPP-NEXT: v_mov_b32_e32 v0, s11
; GFX9_DPP-NEXT: v_subrev_co_u32_e32 v8, vcc, s10, v10
; GFX9_DPP-NEXT: v_subb_co_u32_e32 v9, vcc, v11, v0, vcc
; GFX9_DPP-NEXT: v_swap_b32 v6, v8
@@ -14148,29 +14174,29 @@ define amdgpu_kernel void @uniform_fadd_f16(ptr addrspace(1) %result, ptr addrsp
; GFX7LESS-LABEL: uniform_fadd_f16:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s11, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, s6
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, s11
; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v0
; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
@@ -14919,29 +14945,29 @@ define amdgpu_kernel void @uniform_fadd_bf16(ptr addrspace(1) %result, ptr addrs
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_lshl_b32 s11, s6, 16
+; GFX7LESS-NEXT: s_lshl_b32 s6, s6, 16
; GFX7LESS-NEXT: s_and_b32 s4, s2, -4
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX7LESS-NEXT: s_and_b32 s2, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s10, s2, 3
; GFX7LESS-NEXT: s_lshl_b32 s2, 0xffff, s10
+; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s6
; GFX7LESS-NEXT: s_not_b32 s2, s2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: v_mov_b32_e32 v1, s3
+; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, s10, v1
-; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s11
-; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v0, v0, v4
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_and_b32_e32 v3, s2, v1
+; GFX7LESS-NEXT: s_waitcnt expcnt(0)
+; GFX7LESS-NEXT: v_and_b32_e32 v2, s2, v1
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, s10, v0
-; GFX7LESS-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX7LESS-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
@@ -15471,39 +15497,39 @@ define amdgpu_kernel void @uniform_fadd_v2f16(ptr addrspace(1) %result, ptr addr
; GFX7LESS-LABEL: uniform_fadd_v2f16:
; GFX7LESS: ; %bb.0:
; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7LESS-NEXT: s_load_dword s10, s[4:5], 0xd
+; GFX7LESS-NEXT: s_load_dword s4, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
-; GFX7LESS-NEXT: s_mov_b32 s4, s2
-; GFX7LESS-NEXT: s_lshr_b32 s11, s10, 16
+; GFX7LESS-NEXT: s_lshr_b32 s10, s4, 16
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, s4
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, s10
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_lshr_b32 s2, s5, 16
+; GFX7LESS-NEXT: s_lshr_b32 s4, s5, 16
; GFX7LESS-NEXT: v_mov_b32_e32 v0, s5
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, s4
+; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v2, s11
-; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v3, v1
-; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, s10
+; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX7LESS-NEXT: v_cvt_f32_f16_e32 v5, v0
; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7LESS-NEXT: v_add_f32_e32 v2, v3, v2
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7LESS-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v4, v2
-; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX7LESS-NEXT: v_or_b32_e32 v2, v0, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v3
+; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX7LESS-NEXT: v_cvt_f16_f32_e32 v6, v5
+; GFX7LESS-NEXT: v_or_b32_e32 v5, v0, v1
; GFX7LESS-NEXT: v_lshlrev_b32_e32 v0, 16, v4
-; GFX7LESS-NEXT: v_or_b32_e32 v1, v3, v0
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, v2
+; GFX7LESS-NEXT: v_or_b32_e32 v4, v6, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v0, v4
+; GFX7LESS-NEXT: v_mov_b32_e32 v1, v5
; GFX7LESS-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v1, 16, v0
@@ -15871,50 +15897,50 @@ define amdgpu_kernel void @uniform_fadd_v2bf16(ptr addrspace(1) %result, ptr add
; GFX7LESS-NEXT: s_load_dword s4, s[4:5], 0xd
; GFX7LESS-NEXT: s_mov_b64 s[8:9], 0
; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_load_dword s5, s[2:3], 0x0
-; GFX7LESS-NEXT: s_and_b32 s10, s4, 0xffff0000
-; GFX7LESS-NEXT: s_lshl_b32 s11, s4, 16
+; GFX7LESS-NEXT: s_and_b32 s6, s4, 0xffff0000
+; GFX7LESS-NEXT: s_lshl_b32 s4, s4, 16
+; GFX7LESS-NEXT: v_mul_f32_e64 v0, 1.0, s4
+; GFX7LESS-NEXT: v_mul_f32_e64 v1, 1.0, s6
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_and_b32 s4, s5, 0xffff0000
; GFX7LESS-NEXT: s_lshl_b32 s5, s5, 16
-; GFX7LESS-NEXT: v_mov_b32_e32 v1, s5
-; GFX7LESS-NEXT: v_mov_b32_e32 v0, s4
+; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7LESS-NEXT: v_mov_b32_e32 v3, s5
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s4
+; GFX7LESS-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_mov_b32 s4, s2
; GFX7LESS-NEXT: s_mov_b32 s5, s3
; GFX7LESS-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX7LESS-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7LESS-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7LESS-NEXT: v_mul_f32_e32 v3, 1.0, v3
; GFX7LESS-NEXT: s_waitcnt expcnt(0)
-; GFX7LESS-NEXT: v_mul_f32_e64 v3, 1.0, s10
-; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7LESS-NEXT: v_mul_f32_e64 v2, 1.0, s11
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX7LESS-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX7LESS-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX7LESS-NEXT: v_alignbit_b32 v1, v0, v1, 16
-; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX7LESS-NEXT: v_alignbit_b32 v0, v0, v2, 16
-; GFX7LESS-NEXT: v_mov_b32_e32 v3, v1
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, v0
-; GFX7LESS-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; GFX7LESS-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX7LESS-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7LESS-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX7LESS-NEXT: v_add_f32_e32 v4, v4, v0
+; GFX7LESS-NEXT: v_alignbit_b32 v3, v2, v3, 16
+; GFX7LESS-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX7LESS-NEXT: v_alignbit_b32 v2, v2, v4, 16
+; GFX7LESS-NEXT: v_mov_b32_e32 v5, v3
+; GFX7LESS-NEXT: v_mov_b32_e32 v4, v2
+; GFX7LESS-NEXT: buffer_atomic_cmpswap v[4:5], off, s[4:7], 0 glc
; GFX7LESS-NEXT: s_waitcnt vmcnt(0)
-; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; GFX7LESS-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
+; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX7LESS-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
; GFX7LESS-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7LESS-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7LESS-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX7LESS-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_cbranch_execnz .LBB21_1
; GFX7LESS-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v0
+; GFX7LESS-NEXT: v_mul_f32_e32 v0, 1.0, v2
; GFX7LESS-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v1
+; GFX7LESS-NEXT: v_mul_f32_e32 v1, 1.0, v3
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
; GFX7LESS-NEXT: s_mov_b32 s2, -1
; GFX7LESS-NEXT: v_alignbit_b32 v0, v0, v1, 16
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index ad7db0bf81dae..b813e5b450e0a 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -2378,17 +2378,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: add_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB6_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -2426,17 +2428,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: add_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB6_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB6_1
@@ -2472,17 +2476,19 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: add_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB6_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB6_1
@@ -2516,6 +2522,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: add_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -2523,10 +2530,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s6
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s6
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s6
; GFX1064_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, s8
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB6_1
@@ -2561,6 +2569,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: add_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -2568,10 +2577,11 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB6_1
@@ -2605,7 +2615,8 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: add_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -2613,11 +2624,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s6, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s6
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v2, s6
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s6
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -2655,7 +2667,7 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: add_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -2663,11 +2675,12 @@ define amdgpu_kernel void @add_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
@@ -3421,14 +3434,16 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX7LESS_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX7LESS_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_nop 0
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[4:5], s[2:3], 0
@@ -3454,13 +3469,15 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX8_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
+; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
@@ -3484,13 +3501,15 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX9_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
-; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s4
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s4
+; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
@@ -3512,14 +3531,16 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1064_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s4, s[2:3]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
; GFX1064_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[4:5]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
@@ -3542,14 +3563,16 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1032_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1032_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
; GFX1032_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB7_1
@@ -3571,6 +3594,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1164_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v1, 0
; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
@@ -3579,8 +3603,9 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s4, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s5, v0, s4
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v1, s4
; GFX1164_ITERATIVE-NEXT: s_add_u32 s0, s0, s5
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, s6
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[4:5], 1, s4
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
@@ -3605,7 +3630,7 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
;
; GFX1132_ITERATIVE-LABEL: add_i64_varying_nouse:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1132_ITERATIVE-NEXT: .LBB7_1: ; %ComputeLoop
@@ -3613,8 +3638,9 @@ define amdgpu_kernel void @add_i64_varying_nouse() {
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s4, v0, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s5, v1, s3
; GFX1132_ITERATIVE-NEXT: s_add_u32 s0, s0, s4
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, s5
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
@@ -6499,17 +6525,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: sub_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB14_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX7LESS_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX7LESS_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -6547,17 +6575,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: sub_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB14_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX8_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX8_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB14_1
@@ -6593,17 +6623,19 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: sub_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB14_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s6
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s8, v0, s6
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
+; GFX9_ITERATIVE-NEXT: s_add_u32 s0, s0, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
-; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX9_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB14_1
@@ -6637,6 +6669,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: sub_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -6644,10 +6677,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s6, s[2:3]
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v0, s6
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s8, v3, s6
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s6
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s6
; GFX1064_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
-; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_addc_u32 s1, s1, s8
; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB14_1
@@ -6682,6 +6716,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: sub_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
@@ -6689,10 +6724,11 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
; GFX1032_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1032_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB14_1
@@ -6726,7 +6762,8 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: sub_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -6734,11 +6771,12 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s6, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s6
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s6
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s8, v2, s6
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s6
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s6
; GFX1164_ITERATIVE-NEXT: s_add_u32 s0, s0, s7
-; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_addc_u32 s1, s1, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s6
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
@@ -6776,7 +6814,7 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: sub_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
@@ -6784,11 +6822,12 @@ define amdgpu_kernel void @sub_i64_varying(ptr addrspace(1) %out) {
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
; GFX1132_ITERATIVE-NEXT: s_add_u32 s0, s0, s6
-; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, 0
+; GFX1132_ITERATIVE-NEXT: s_addc_u32 s1, s1, s7
; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
@@ -8277,13 +8316,14 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: and_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
@@ -8324,19 +8364,20 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: and_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX8_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB16_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8369,19 +8410,20 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: and_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX9_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB16_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8412,13 +8454,14 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: and_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s8
@@ -8457,19 +8500,20 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: and_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s7
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB16_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -8501,16 +8545,17 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: and_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
@@ -8550,21 +8595,21 @@ define amdgpu_kernel void @and_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: and_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132_ITERATIVE-NEXT: .LBB16_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1132_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s7
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1132_ITERATIVE-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7]
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB16_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -9881,13 +9926,14 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: or_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
@@ -9928,19 +9974,20 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: or_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX8_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB18_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -9973,19 +10020,20 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: or_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX9_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB18_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -10016,13 +10064,14 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: or_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s8
@@ -10061,19 +10110,20 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: or_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s7
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB18_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -10105,16 +10155,17 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: or_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
@@ -10154,21 +10205,21 @@ define amdgpu_kernel void @or_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: or_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132_ITERATIVE-NEXT: .LBB18_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1132_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s7
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1132_ITERATIVE-NEXT: s_or_b64 s[0:1], s[0:1], s[6:7]
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB18_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -11485,13 +11536,14 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: xor_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
@@ -11532,19 +11584,20 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: xor_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX8_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX8_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX8_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[6:7]
-; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB20_1
; GFX8_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX8_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -11577,19 +11630,20 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: xor_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX9_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX9_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
; GFX9_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[6:7]
-; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
-; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
+; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB20_1
; GFX9_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX9_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -11620,13 +11674,14 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: xor_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s8
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s8
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s8
@@ -11665,19 +11720,20 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: xor_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s7
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[6:7]
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB20_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -11709,16 +11765,17 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: xor_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1164_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s8, s[2:3]
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s8
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s8
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s8
; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s8
@@ -11758,21 +11815,21 @@ define amdgpu_kernel void @xor_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: xor_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX1132_ITERATIVE-NEXT: .LBB20_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s7, 1, s7
-; GFX1132_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s7
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1132_ITERATIVE-NEXT: s_xor_b64 s[0:1], s[0:1], s[6:7]
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB20_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -13390,22 +13447,24 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: max_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[4:5]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX7LESS_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -13445,22 +13504,24 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: max_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX8_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX8_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[4:5]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
@@ -13498,22 +13559,24 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: max_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_brev_b32 s1, 1
; GFX9_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX9_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: v_cmp_gt_i64_e32 vcc, s[0:1], v[4:5]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
@@ -13549,23 +13612,24 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: max_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_brev_b32 s1, 1
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s0, s7
+; GFX1064_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s10, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s10
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -13599,23 +13663,24 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: max_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_brev_b32 s1, 1
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, s3
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[2:3]
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s2
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -13648,26 +13713,28 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: max_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_brev_b32 s1, 1
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s0, s7
+; GFX1164_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164_ITERATIVE-NEXT: .p2align 6
; GFX1164_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s10, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s10
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s10
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
+; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[8:9]
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -13702,26 +13769,27 @@ define amdgpu_kernel void @max_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: max_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_brev_b32 s1, 1
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, s3
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1132_ITERATIVE-NEXT: .p2align 6
; GFX1132_ITERATIVE-NEXT: .LBB23_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[2:3]
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: v_cmp_gt_i64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s2
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB23_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -15584,22 +15652,24 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: min_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 s0, -1
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX7LESS_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -15639,22 +15709,24 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: min_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX8_ITERATIVE-NEXT: s_mov_b32 s0, -1
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX8_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
@@ -15692,22 +15764,24 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: min_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX9_ITERATIVE-NEXT: s_mov_b32 s0, -1
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX9_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[3:4]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
@@ -15743,23 +15817,24 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: min_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX1064_ITERATIVE-NEXT: s_mov_b32 s0, -1
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s10, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s10
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -15793,23 +15868,24 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: min_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX1032_ITERATIVE-NEXT: s_mov_b32 s0, -1
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[2:3]
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s2
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -15842,26 +15918,28 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: min_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX1164_ITERATIVE-NEXT: s_mov_b32 s0, -1
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164_ITERATIVE-NEXT: .p2align 6
; GFX1164_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s10, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s10
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s10
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
+; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[8:9]
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -15896,26 +15974,27 @@ define amdgpu_kernel void @min_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: min_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_brev_b32 s1, -2
; GFX1132_ITERATIVE-NEXT: s_mov_b32 s0, -1
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1132_ITERATIVE-NEXT: .p2align 6
; GFX1132_ITERATIVE-NEXT: .LBB26_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[2:3]
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: v_cmp_lt_i64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s2
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB26_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -17773,21 +17852,23 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: umax_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX7LESS_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -17827,21 +17908,23 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: umax_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX8_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
@@ -17879,21 +17962,23 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: umax_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX9_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: v_cmp_gt_u64_e32 vcc, s[0:1], v[4:5]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
@@ -17929,22 +18014,23 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: umax_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s10, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s10
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -17978,22 +18064,23 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: umax_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[2:3]
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s2
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -18026,25 +18113,27 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: umax_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164_ITERATIVE-NEXT: .p2align 6
; GFX1164_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s10, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s10
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s10
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
+; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[8:9]
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -18079,25 +18168,26 @@ define amdgpu_kernel void @umax_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: umax_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], 0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1132_ITERATIVE-NEXT: .p2align 6
; GFX1132_ITERATIVE-NEXT: .LBB29_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[2:3]
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_cmp_gt_u64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s2
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB29_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -19950,21 +20040,23 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX7LESS_ITERATIVE-LABEL: umin_i64_varying:
; GFX7LESS_ITERATIVE: ; %bb.0: ; %entry
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX7LESS_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
; GFX7LESS_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX7LESS_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX7LESS_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7LESS_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX7LESS_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX7LESS_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX7LESS_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX7LESS_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[4:5]
; GFX7LESS_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX7LESS_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX7LESS_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX7LESS_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX7LESS_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX7LESS_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX7LESS_ITERATIVE-NEXT: v_cmp_ne_u64_e64 s[6:7], s[2:3], 0
@@ -20004,21 +20096,23 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX8_ITERATIVE-LABEL: umin_i64_varying:
; GFX8_ITERATIVE: ; %bb.0: ; %entry
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX8_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
; GFX8_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX8_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX8_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX8_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX8_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX8_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX8_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[4:5]
; GFX8_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX8_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX8_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX8_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX8_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX8_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX8_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
@@ -20056,21 +20150,23 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
; GFX9_ITERATIVE-LABEL: umin_i64_varying:
; GFX9_ITERATIVE: ; %bb.0: ; %entry
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, 0
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX9_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
; GFX9_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX9_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX9_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9_ITERATIVE-NEXT: s_ff1_i32_b64 s8, s[2:3]
-; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v0, s8
-; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v3, s9
-; GFX9_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[3:4]
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s9, v3, s8
+; GFX9_ITERATIVE-NEXT: v_readlane_b32 s10, v0, s8
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v4, s10
+; GFX9_ITERATIVE-NEXT: v_mov_b32_e32 v5, s9
+; GFX9_ITERATIVE-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[4:5]
; GFX9_ITERATIVE-NEXT: s_mov_b32 m0, s8
; GFX9_ITERATIVE-NEXT: s_and_b64 s[6:7], vcc, exec
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v2, s1, m0
; GFX9_ITERATIVE-NEXT: v_writelane_b32 v1, s0, m0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s9
+; GFX9_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s10
; GFX9_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s8
; GFX9_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX9_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
@@ -20106,22 +20202,23 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1064_ITERATIVE-LABEL: umin_i64_varying:
; GFX1064_ITERATIVE: ; %bb.0: ; %entry
+; GFX1064_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1064_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1064_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1064_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1064_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX1064_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1064_ITERATIVE-NEXT: s_ff1_i32_b64 s10, s[2:3]
+; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s10
; GFX1064_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s10
; GFX1064_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s10
; GFX1064_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1064_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
-; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[8:9]
+; GFX1064_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
+; GFX1064_ITERATIVE-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
; GFX1064_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX1064_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1064_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -20155,22 +20252,23 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1032_ITERATIVE-LABEL: umin_i64_varying:
; GFX1032_ITERATIVE: ; %bb.0: ; %entry
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1032_ITERATIVE-NEXT: v_mov_b32_e32 v3, 0
+; GFX1032_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1032_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1032_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1032_ITERATIVE-NEXT: ; implicit-def: $vgpr1_vgpr2
; GFX1032_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX1032_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s7, s6
-; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s2, v0, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s7
-; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s7
-; GFX1032_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[2:3]
+; GFX1032_ITERATIVE-NEXT: s_ff1_i32_b32 s3, s2
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s7, v3, s3
+; GFX1032_ITERATIVE-NEXT: v_readlane_b32 s6, v0, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v2, s1, s3
+; GFX1032_ITERATIVE-NEXT: v_writelane_b32 v1, s0, s3
+; GFX1032_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
; GFX1032_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s6, s6, s2
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1032_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1032_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1032_ITERATIVE-NEXT: s_andn2_b32 s2, s2, s3
; GFX1032_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX1032_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1032_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v0, exec_lo, 0
@@ -20203,25 +20301,27 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1164_ITERATIVE-LABEL: umin_i64_varying:
; GFX1164_ITERATIVE: ; %bb.0: ; %entry
-; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
+; GFX1164_ITERATIVE-NEXT: v_mov_b32_e32 v2, 0
+; GFX1164_ITERATIVE-NEXT: v_and_b32_e32 v3, 0x3ff, v0
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[2:3], exec
; GFX1164_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1164_ITERATIVE-NEXT: s_mov_b32 s7, 0
; GFX1164_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1164_ITERATIVE-NEXT: .p2align 6
; GFX1164_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX1164_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1164_ITERATIVE-NEXT: s_ctz_i32_b64 s10, s[2:3]
-; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v2, s10
+; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s10
+; GFX1164_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s10
; GFX1164_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s10
; GFX1164_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s[8:9], s[0:1], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
+; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
; GFX1164_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
-; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[8:9], 1, s10
+; GFX1164_ITERATIVE-NEXT: s_lshl_b64 s[6:7], 1, s10
; GFX1164_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[8:9]
+; GFX1164_ITERATIVE-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[6:7]
; GFX1164_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX1164_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1164_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -20256,25 +20356,26 @@ define amdgpu_kernel void @umin_i64_varying(ptr addrspace(1) %out) {
;
; GFX1132_ITERATIVE-LABEL: umin_i64_varying:
; GFX1132_ITERATIVE: ; %bb.0: ; %entry
-; GFX1132_ITERATIVE-NEXT: v_and_b32_e32 v2, 0x3ff, v0
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132_ITERATIVE-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v3, 0x3ff, v0
+; GFX1132_ITERATIVE-NEXT: s_mov_b32 s2, exec_lo
; GFX1132_ITERATIVE-NEXT: s_mov_b64 s[0:1], -1
-; GFX1132_ITERATIVE-NEXT: s_mov_b32 s3, 0
; GFX1132_ITERATIVE-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX1132_ITERATIVE-NEXT: .p2align 6
; GFX1132_ITERATIVE-NEXT: .LBB32_1: ; %ComputeLoop
; GFX1132_ITERATIVE-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s7, s6
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s2, v2, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s7
-; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s7
-; GFX1132_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[2:3]
+; GFX1132_ITERATIVE-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s7, v2, s3
+; GFX1132_ITERATIVE-NEXT: v_readlane_b32 s6, v3, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v1, s1, s3
+; GFX1132_ITERATIVE-NEXT: v_writelane_b32 v0, s0, s3
+; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1132_ITERATIVE-NEXT: v_cmp_lt_u64_e64 s8, s[0:1], s[6:7]
; GFX1132_ITERATIVE-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, 0
-; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s2
-; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s2, 1, s7
-; GFX1132_ITERATIVE-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s6, s6, s2
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s1, s1, s7
+; GFX1132_ITERATIVE-NEXT: s_cselect_b32 s0, s0, s6
+; GFX1132_ITERATIVE-NEXT: s_lshl_b32 s3, 1, s3
+; GFX1132_ITERATIVE-NEXT: s_and_not1_b32 s2, s2, s3
; GFX1132_ITERATIVE-NEXT: s_cbranch_scc1 .LBB32_1
; GFX1132_ITERATIVE-NEXT: ; %bb.2: ; %ComputeEnd
; GFX1132_ITERATIVE-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
index bbc9ccdf23b2c..292b3de7ca71e 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
@@ -8,29 +8,27 @@ define amdgpu_kernel void @flat_atomic_usub_cond_no_rtn_u32(ptr %addr, i32 %in)
; GFX9-SDAG-LABEL: flat_atomic_usub_cond_no_rtn_u32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_add_u32 s0, s0, -16
; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, -1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-SDAG-NEXT: flat_load_dword v1, v[0:1]
+; GFX9-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
; GFX9-SDAG-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-SDAG-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v2, s2, v3
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v3
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-SDAG-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
@@ -52,29 +50,27 @@ define amdgpu_kernel void @flat_atomic_usub_cond_no_rtn_u32(ptr %addr, i32 %in)
; GFX9-GISEL-LABEL: flat_atomic_usub_cond_no_rtn_u32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, -16
; GFX9-GISEL-NEXT: s_addc_u32 s1, s1, -1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-GISEL-NEXT: flat_load_dword v1, v[0:1]
+; GFX9-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX9-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX9-GISEL-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-GISEL-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, s2, v3
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s2, v3
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GFX9-GISEL-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX9-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB0_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
@@ -414,19 +410,18 @@ define amdgpu_kernel void @ds_usub_cond_no_rtn_u32(ptr addrspace(3) %addr, i32 %
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_add_i32 s0, s0, -16
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-SDAG-NEXT: ds_read_b32 v0, v0
+; GFX9-SDAG-NEXT: ds_read_b32 v1, v0
; GFX9-SDAG-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s1, v0
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s1, v0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
-; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v1, v2, v0, v1
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v2, s1, v1
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s1, v1
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
+; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v0
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v2
; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -451,21 +446,20 @@ define amdgpu_kernel void @ds_usub_cond_no_rtn_u32(ptr addrspace(3) %addr, i32 %
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_add_u32 s0, s0, -16
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-GISEL-NEXT: ds_read_b32 v0, v0
+; GFX9-GISEL-NEXT: ds_read_b32 v1, v0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-GISEL-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s1, v0
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s1, v0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
-; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v1, v2, v0, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, s1, v1
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s1, v1
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
+; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v2, v0, v1, v2
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v0
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v2
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -495,25 +489,24 @@ define amdgpu_kernel void @ds_usub_cond_rtn_u32(ptr addrspace(3) %addr, i32 %in,
; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-SDAG-NEXT: ds_read_b32 v0, v0 offset:16
+; GFX9-SDAG-NEXT: ds_read_b32 v1, v0 offset:16
; GFX9-SDAG-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v2, s1, v1
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s1, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 offset:16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s1, v2
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s1, v2
+; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-SDAG-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:16
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-SDAG-NEXT: ds_write_b32 v1, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: ds_write_b32 v0, v1
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX12-SDAG-LABEL: ds_usub_cond_rtn_u32:
@@ -535,27 +528,26 @@ define amdgpu_kernel void @ds_usub_cond_rtn_u32(ptr addrspace(3) %addr, i32 %in,
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-GISEL-NEXT: ds_read_b32 v0, v0 offset:16
+; GFX9-GISEL-NEXT: ds_read_b32 v1, v0 offset:16
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_readfirstlane_b32 s4, v0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0
; GFX9-GISEL-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, s1, v1
-; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s1, v1
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v2, v1, v2, vcc
-; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v0, v0, v1, v2 offset:16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, s1, v2
+; GFX9-GISEL-NEXT: v_cmp_le_u32_e32 vcc, s1, v2
+; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX9-GISEL-NEXT: ds_cmpst_rtn_b32 v1, v0, v2, v1 offset:16
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-GISEL-NEXT: ds_write_b32 v1, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-GISEL-NEXT: ds_write_b32 v0, v1
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: ds_usub_cond_rtn_u32:
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index 83f834291e0ec..eb9cc4a7b524d 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -1683,8 +1683,8 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1693,8 +1693,8 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v3, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v1, v2
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v0, v[2:3], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v0, v2
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v1, v[2:3], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
@@ -1957,8 +1957,8 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1966,8 +1966,8 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
; GFX9-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s4 clamp
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v3, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:2048 glc
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v2, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -3173,12 +3173,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffffff00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s5
; GFX9-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
@@ -3186,9 +3186,9 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v4
; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, s4 clamp
-; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v2, v3
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v0, v[3:4], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v1, v3
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v2, v[3:4], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
@@ -3472,21 +3472,21 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffffff00
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xffffff00
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
; GFX9-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v1
; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v0, s4 clamp
-; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
-; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v4, v0
-; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
+; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v3, v0
+; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v4, v[0:1], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 943cd2c84896a..fbcba25ba207c 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -5,17 +5,17 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-LABEL: vgpr_mfma_pass_av_split_crash:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
+; CHECK-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10
; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
-; CHECK-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x10
; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_bitcmp1_b32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[10:11], -1, 0
-; CHECK-NEXT: s_xor_b64 s[12:13], s[10:11], -1
-; CHECK-NEXT: s_bitcmp1_b32 s0, 8
; CHECK-NEXT: s_cselect_b64 s[14:15], -1, 0
; CHECK-NEXT: s_xor_b64 s[16:17], s[14:15], -1
+; CHECK-NEXT: s_bitcmp1_b32 s0, 8
+; CHECK-NEXT: s_cselect_b64 s[18:19], -1, 0
+; CHECK-NEXT: s_xor_b64 s[20:21], s[18:19], -1
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
; CHECK-NEXT: v_mov_b32_e32 v4, 0x19f4ec90
@@ -28,13 +28,16 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v11, 0xbf8c6ea4
; CHECK-NEXT: v_mov_b32_e32 v12, 0xe82d3ff0
; CHECK-NEXT: v_mov_b32_e32 v13, 0xbfa59976
-; CHECK-NEXT: s_mov_b64 s[18:19], 0
+; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
+; CHECK-NEXT: s_and_b64 s[0:1], exec, s[14:15]
; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
+; CHECK-NEXT: s_and_b64 s[2:3], exec, s[18:19]
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
+; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
@@ -42,24 +45,24 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_branch .LBB0_2
; CHECK-NEXT: .LBB0_1: ; %Flow9
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_and_b64 s[0:1], s[2:3], exec
-; CHECK-NEXT: s_cselect_b32 s0, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s0, 1
+; CHECK-NEXT: s_and_b64 s[6:7], s[24:25], exec
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s6, 1
; CHECK-NEXT: s_cbranch_scc0 .LBB0_17
; CHECK-NEXT: .LBB0_2: ; %._crit_edge1942.i.i.i3548
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB0_6 Depth 2
-; CHECK-NEXT: s_and_b64 s[0:1], s[14:15], exec
-; CHECK-NEXT: s_cselect_b32 s2, 1, 0
-; CHECK-NEXT: s_cmp_lg_u32 s2, 1
+; CHECK-NEXT: s_and_b64 s[6:7], exec, s[18:19]
+; CHECK-NEXT: s_cselect_b32 s6, 1, 0
+; CHECK-NEXT: s_cmp_lg_u32 s6, 1
; CHECK-NEXT: s_cbranch_scc1 .LBB0_9
; CHECK-NEXT: ; %bb.3: ; %.preheader1868.i.i.i3244
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[10:11]
+; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
; CHECK-NEXT: s_cbranch_vccz .LBB0_10
; CHECK-NEXT: ; %bb.4: ; %.preheader1855.i.i.i3329.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[6:7]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
@@ -87,62 +90,62 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_branch .LBB0_6
; CHECK-NEXT: .LBB0_5: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[22:23]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_11
; CHECK-NEXT: .LBB0_6: ; %.preheader1855.i.i.i3329
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; => This Inner Loop Header: Depth=2
; CHECK-NEXT: v_accvgpr_read_b32 v27, a3
; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
-; CHECK-NEXT: s_mov_b64 s[20:21], -1
-; CHECK-NEXT: s_mov_b64 s[22:23], -1
-; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
+; CHECK-NEXT: s_mov_b64 s[24:25], -1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
+; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
; CHECK-NEXT: ; implicit-def: $agpr2_agpr3
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: ; %bb.7: ; %.lr.ph2070.i.i.i3291
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[12:13]
; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
; CHECK-NEXT: v_accvgpr_mov_b32 a2, a0
-; CHECK-NEXT: s_mov_b64 s[22:23], s[12:13]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[16:17]
+; CHECK-NEXT: s_mov_b64 vcc, s[4:5]
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
-; CHECK-NEXT: s_mov_b64 s[20:21], 0
+; CHECK-NEXT: s_mov_b64 s[24:25], 0
; CHECK-NEXT: v_accvgpr_write_b32 a2, v28
; CHECK-NEXT: v_accvgpr_write_b32 a3, v29
-; CHECK-NEXT: s_mov_b64 s[22:23], 0
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[8:9]
-; CHECK-NEXT: s_mov_b64 s[18:19], 0
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[12:13]
+; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
-; CHECK-NEXT: s_mov_b64 s[0:1], s[16:17]
+; CHECK-NEXT: s_mov_b64 s[6:7], s[20:21]
; CHECK-NEXT: s_branch .LBB0_15
; CHECK-NEXT: .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
; CHECK-NEXT: s_branch .LBB0_15
; CHECK-NEXT: .LBB0_11: ; %loop.exit.guard
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[20:21]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[24:25]
; CHECK-NEXT: s_cbranch_vccz .LBB0_13
; CHECK-NEXT: ; %bb.12: ; %._crit_edge2105.i.i.i2330.loopexit
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_cmp_nlg_f64_e64 s[0:1], 0, v[26:27]
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[10:11]
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[10:11]
-; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[0:1]
+; CHECK-NEXT: v_cmp_nlg_f64_e64 s[6:7], 0, v[26:27]
+; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[6:7]
; CHECK-NEXT: v_mov_b32_e32 v27, v26
-; CHECK-NEXT: s_and_b64 s[0:1], exec, s[10:11]
-; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[4:5]
-; CHECK-NEXT: s_cselect_b32 s19, s19, 0
-; CHECK-NEXT: s_cselect_b32 s18, s18, 0
-; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: s_and_b64 s[6:7], exec, s[14:15]
+; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[8:9]
+; CHECK-NEXT: s_cselect_b32 s23, s23, 0
+; CHECK-NEXT: s_cselect_b32 s22, s22, 0
+; CHECK-NEXT: s_mov_b64 s[6:7], -1
; CHECK-NEXT: s_branch .LBB0_14
; CHECK-NEXT: .LBB0_13: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[0:1], 0
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
; CHECK-NEXT: .LBB0_14: ; %Flow6
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
@@ -150,13 +153,13 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
; CHECK-NEXT: .LBB0_15: ; %Flow6
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[2:3], -1
-; CHECK-NEXT: s_and_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_mov_b64 s[24:25], -1
+; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccz .LBB0_1
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: s_mov_b64 s[2:3], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[4:5]
+; CHECK-NEXT: s_mov_b64 s[24:25], 0
+; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[8:9]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index 0cc2d8b06d254..4719d6edbaaa5 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -57,22 +57,22 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB0_1
@@ -96,18 +96,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB0_1
@@ -122,18 +122,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
@@ -148,18 +148,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB0_1
@@ -175,19 +175,19 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB0_1
@@ -241,22 +241,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB1_1
@@ -288,19 +288,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB1_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -313,19 +313,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB1_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -339,20 +339,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: v_mov_b32_e32 v4, v1
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB1_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -813,22 +813,22 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
@@ -843,12 +843,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -868,18 +868,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
@@ -894,18 +894,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
@@ -920,18 +920,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB3_1
@@ -947,19 +947,19 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_fine_g
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB3_1
@@ -1013,22 +1013,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB4_1
@@ -1046,7 +1046,6 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_add_f32_e32 v2, v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1066,19 +1065,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1091,19 +1090,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1116,19 +1115,19 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB4_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1142,20 +1141,20 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f32_e32 v1, v2, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: v_mov_b32_e32 v4, v1
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB4_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1196,23 +1195,23 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1226,22 +1225,22 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB5_1
@@ -1256,12 +1255,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1281,18 +1280,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
@@ -1307,18 +1306,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
@@ -1333,18 +1332,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB5_1
@@ -1360,19 +1359,19 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset(ptr addrspace(7)
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB5_1
@@ -1414,23 +1413,23 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1444,22 +1443,22 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB6_1
@@ -1474,12 +1473,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1499,18 +1498,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB6_1
@@ -1525,18 +1524,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
@@ -1551,18 +1550,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB6_1
@@ -1578,19 +1577,19 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB6_1
@@ -1632,23 +1631,23 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote_memory__amdgpu_ignore_denormal_mode:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1662,22 +1661,22 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB7_1
@@ -1692,12 +1691,12 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1717,18 +1716,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
@@ -1743,18 +1742,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
@@ -1769,18 +1768,18 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB7_1
@@ -1796,19 +1795,19 @@ define float @buffer_fat_ptr_agent_atomic_fadd_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB7_1
@@ -1835,18 +1834,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX12-NEXT: v_mov_b32_e32 v1, v7
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -1876,18 +1874,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_mov_b32_e32 v2, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX11-NEXT: v_mov_b32_e32 v1, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
@@ -1908,13 +1905,13 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1951,11 +1948,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: v_mov_b32_e32 v0, v6
@@ -1981,11 +1978,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: v_mov_b32_e32 v0, v6
@@ -2011,14 +2008,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
-; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -2039,18 +2036,18 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
-; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -2079,22 +2076,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[2:3], v[4:5], v[0:1]
-; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: v_mov_b32_e32 v8, v4
+; GFX12-NEXT: v_dual_mov_b32 v10, v5 :: v_dual_mov_b32 v9, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v4, v7 :: v_dual_mov_b32 v5, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2118,23 +2115,23 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX11-NEXT: v_mov_b32_e32 v8, v4
+; GFX11-NEXT: v_dual_mov_b32 v10, v5 :: v_dual_mov_b32 v9, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v8, v3 :: v_dual_mov_b32 v7, v2
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v4, v7 :: v_dual_mov_b32 v5, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2147,25 +2144,25 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: v_mov_b32_e32 v6, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
-; GFX10-NEXT: v_mov_b32_e32 v9, v5
-; GFX10-NEXT: v_mov_b32_e32 v8, v4
+; GFX10-NEXT: v_mov_b32_e32 v10, v5
+; GFX10-NEXT: v_mov_b32_e32 v9, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v7, v3
-; GFX10-NEXT: v_mov_b32_e32 v6, v2
-; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX10-NEXT: v_mov_b32_e32 v8, v3
+; GFX10-NEXT: v_mov_b32_e32 v7, v2
+; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX10-NEXT: v_mov_b32_e32 v4, v6
-; GFX10-NEXT: v_mov_b32_e32 v5, v7
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v4, v7
+; GFX10-NEXT: v_mov_b32_e32 v5, v8
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB9_1
@@ -2188,22 +2185,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_mov_b32_e32 v9, v5
-; GFX908-NEXT: v_mov_b32_e32 v8, v4
-; GFX908-NEXT: v_mov_b32_e32 v7, v3
-; GFX908-NEXT: v_mov_b32_e32 v6, v2
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v10, v5
+; GFX908-NEXT: v_mov_b32_e32 v9, v4
+; GFX908-NEXT: v_mov_b32_e32 v8, v3
+; GFX908-NEXT: v_mov_b32_e32 v7, v2
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v6
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v4, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v7
+; GFX908-NEXT: v_mov_b32_e32 v5, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2216,22 +2213,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: v_mov_b32_e32 v2, s20
; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_mov_b32_e32 v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v8, v4
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_mov_b32_e32 v10, v5
+; GFX8-NEXT: v_mov_b32_e32 v9, v4
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v6
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v4, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v7
+; GFX8-NEXT: v_mov_b32_e32 v5, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2244,22 +2241,22 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s20
; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
-; GFX7-NEXT: v_mov_b32_e32 v9, v5
-; GFX7-NEXT: v_mov_b32_e32 v8, v4
-; GFX7-NEXT: v_mov_b32_e32 v7, v3
-; GFX7-NEXT: v_mov_b32_e32 v6, v2
-; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX7-NEXT: v_mov_b32_e32 v10, v5
+; GFX7-NEXT: v_mov_b32_e32 v9, v4
+; GFX7-NEXT: v_mov_b32_e32 v8, v3
+; GFX7-NEXT: v_mov_b32_e32 v7, v2
+; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v4, v6
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v4, v7
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v5, v7
+; GFX7-NEXT: v_mov_b32_e32 v5, v8
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2273,23 +2270,23 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v6, s6
; GFX6-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f64 v[2:3], v[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v9, v5
-; GFX6-NEXT: v_mov_b32_e32 v8, v4
-; GFX6-NEXT: v_mov_b32_e32 v7, v3
-; GFX6-NEXT: v_mov_b32_e32 v6, v2
-; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v10, v5
+; GFX6-NEXT: v_mov_b32_e32 v9, v4
+; GFX6-NEXT: v_mov_b32_e32 v8, v3
+; GFX6-NEXT: v_mov_b32_e32 v7, v2
+; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v4, v6
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v4, v7
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v5, v7
+; GFX6-NEXT: v_mov_b32_e32 v5, v8
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB9_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2824,18 +2821,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX12-NEXT: v_mov_b32_e32 v1, v7
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -2865,18 +2861,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_mov_b32_e32 v2, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX11-NEXT: v_mov_b32_e32 v1, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
@@ -2897,13 +2892,13 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -2931,11 +2926,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX90A-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: v_mov_b32_e32 v5, v1
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v10, s20
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v10, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
@@ -2958,11 +2953,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: v_mov_b32_e32 v0, v6
@@ -2988,11 +2983,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: v_mov_b32_e32 v0, v6
@@ -3018,14 +3013,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
-; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -3046,18 +3041,18 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_remot
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
-; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -3087,18 +3082,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v10, s16
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_add_f64_e32 v[6:7], v[8:9], v[4:5]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX12-NEXT: v_mov_b32_e32 v1, v7
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
@@ -3128,18 +3122,17 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v10, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_mov_b32_e32 v2, v8
+; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_dual_mov_b32 v3, v9 :: v_dual_mov_b32 v0, v6
-; GFX11-NEXT: v_mov_b32_e32 v1, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
@@ -3160,13 +3153,13 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
; GFX10-NEXT: v_mov_b32_e32 v2, v8
; GFX10-NEXT: v_mov_b32_e32 v3, v9
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -3203,11 +3196,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
; GFX908-NEXT: v_mov_b32_e32 v2, v8
; GFX908-NEXT: v_mov_b32_e32 v3, v9
; GFX908-NEXT: v_mov_b32_e32 v0, v6
@@ -3233,11 +3226,11 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
; GFX8-NEXT: v_mov_b32_e32 v2, v8
; GFX8-NEXT: v_mov_b32_e32 v3, v9
; GFX8-NEXT: v_mov_b32_e32 v0, v6
@@ -3263,14 +3256,14 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v10, s20
; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
+; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v0, v6
; GFX7-NEXT: v_mov_b32_e32 v1, v7
-; GFX7-NEXT: v_mov_b32_e32 v2, v8
; GFX7-NEXT: v_mov_b32_e32 v3, v9
; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
@@ -3291,18 +3284,18 @@ define double @buffer_fat_ptr_agent_atomic_fadd_ret_f64__offset__amdgpu_no_fine_
; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: v_mov_b32_e32 v0, s20
; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_add_f64 v[6:7], v[8:9], v[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v0, v6
; GFX6-NEXT: v_mov_b32_e32 v1, v7
-; GFX6-NEXT: v_mov_b32_e32 v2, v8
; GFX6-NEXT: v_mov_b32_e32 v3, v9
; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
@@ -3337,42 +3330,42 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3385,68 +3378,67 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add_f16_e32 v1, v1, v0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v1, s4
; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, s6, v3
+; GFX942-NEXT: v_add_f16_e32 v2, v2, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, s6, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, s7, v2
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
@@ -3456,94 +3448,91 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f16_e32 v1, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3551,22 +3540,21 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v1, v2, s6, v1
; GFX10-NEXT: v_mov_b32_e32 v4, v2
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
@@ -3575,36 +3563,35 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, s6, v3
+; GFX90A-NEXT: v_add_f16_e32 v2, v2, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, s6, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v2
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
@@ -3614,29 +3601,28 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v5, s4
+; GFX908-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v1, s6, v2
; GFX908-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, s6, v1
+; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v1
; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
@@ -3649,30 +3635,29 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v3
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: buffer_load_dword v2, v5, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, s6, v2
; GFX8-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX8-NEXT: v_and_b32_e32 v3, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
+; GFX8-NEXT: v_and_b32_e32 v3, s7, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, s6, v1
; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
@@ -3685,36 +3670,35 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -3724,37 +3708,36 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -3764,7 +3747,7 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: s_cbranch_execnz .LBB13_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
@@ -3783,41 +3766,41 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3830,67 +3813,66 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_add_f16_e32 v1, v1, v0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v1, s4
; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, s6, v3
+; GFX942-NEXT: v_add_f16_e32 v2, v2, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, s6, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, s7, v2
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
@@ -3906,85 +3888,82 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.h, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
-; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_f16_e32 v1, v1, v0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -3992,59 +3971,57 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, s6, v3
+; GFX90A-NEXT: v_add_f16_e32 v2, v2, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, s6, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s7, v2
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
@@ -4060,30 +4037,29 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v3, s4
+; GFX908-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v1, s6, v2
; GFX908-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, s6, v1
+; GFX908-NEXT: v_and_or_b32 v1, v2, s7, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4094,31 +4070,30 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-NEXT: buffer_load_dword v2, v3, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, s6, v2
; GFX8-NEXT: v_add_f16_e32 v1, v1, v0
-; GFX8-NEXT: v_and_b32_e32 v3, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, s6, v1
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4129,34 +4104,33 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4167,35 +4141,34 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_f16__offset__amdgpu_no_fine_
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB14_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5001,27 +4974,27 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -5031,56 +5004,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB16_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v4, s4
+; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v5, s6
-; GFX942-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v5
; GFX942-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX942-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5090,32 +5064,33 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_cbranch_execnz .LBB16_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -5125,97 +5100,95 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v5
; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v5
; GFX90A-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5225,39 +5198,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v5
; GFX908-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX908-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX908-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX908-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5267,41 +5239,40 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: s_cbranch_execnz .LBB16_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v5
; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, s8, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5311,38 +5282,37 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: s_cbranch_execnz .LBB16_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5352,39 +5322,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: s_cbranch_execnz .LBB16_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -5394,7 +5363,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: s_cbranch_execnz .LBB16_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -5411,85 +5380,86 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB17_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v3, s6
-; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v3
; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX942-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -5505,123 +5475,122 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v3
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, v1
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v2, s4
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -5637,37 +5606,36 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX908-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v0, s9
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX908-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX908-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v1
-; GFX908-NEXT: v_mov_b32_e32 v3, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5678,39 +5646,38 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_add_f32_e32 v4, v4, v2
-; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5721,36 +5688,35 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_add_f32_e32 v0, v0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB17_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5761,37 +5727,36 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB17_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6525,27 +6490,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6558,7 +6523,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
@@ -6571,16 +6536,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6601,21 +6566,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB19_1
@@ -6645,17 +6610,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB19_1
@@ -6672,23 +6637,23 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f16_sdwa v0, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v1, v4, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB19_1
@@ -6707,27 +6672,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6750,27 +6715,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6817,6 +6782,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
@@ -6827,15 +6793,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6848,6 +6814,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
@@ -6861,15 +6828,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6889,21 +6856,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB20_1
@@ -6937,6 +6904,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -6944,17 +6912,16 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v3, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6972,27 +6939,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -7012,27 +6979,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -7722,27 +7689,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7755,7 +7722,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
@@ -7768,16 +7735,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX11-FAKE16-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7798,21 +7765,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB22_1
@@ -7833,11 +7800,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -7863,17 +7830,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_1
@@ -7890,23 +7857,23 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f16_sdwa v0, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v1, v4, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
@@ -7925,27 +7892,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -7968,27 +7935,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset(ptr addrsp
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8035,6 +8002,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
@@ -8045,15 +8013,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8066,6 +8034,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
@@ -8079,15 +8048,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8107,21 +8076,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB23_1
@@ -8141,10 +8110,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_pk_add_f16 v2, v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8170,18 +8139,18 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8196,6 +8165,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -8203,17 +8173,16 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v3, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8231,27 +8200,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8271,27 +8240,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset(ptr addrspace(
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8334,27 +8303,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8367,7 +8336,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
@@ -8380,16 +8349,16 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v3, v4, v2
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v4, v5, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8410,21 +8379,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB24_1
@@ -8445,11 +8414,11 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_pk_add_f16 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8475,17 +8444,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_pk_add_f16 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_pk_add_f16 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB24_1
@@ -8502,23 +8471,23 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f16_sdwa v0, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v1, v4, v2
-; GFX8-NEXT: v_or_b32_e32 v3, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f16_sdwa v0, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_f16_e32 v1, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB24_1
@@ -8537,27 +8506,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8580,27 +8549,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fadd_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8647,6 +8616,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
@@ -8657,15 +8627,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8678,6 +8648,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
@@ -8691,15 +8662,15 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8719,21 +8690,21 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_mov_b32_e32 v4, v1
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB25_1
@@ -8753,10 +8724,10 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_pk_add_f16 v2, v3, v0
-; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v1, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -8782,18 +8753,18 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_pk_add_f16 v1, v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v4, v1
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB25_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8808,6 +8779,7 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: s_waitcnt vmcnt(1)
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -8815,17 +8787,16 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_add_f16_sdwa v1, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v3, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_add_f16_e32 v4, v2, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, v1
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8843,27 +8814,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -8883,27 +8854,27 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2f16__offset__amdgpu_no_rem
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_add_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_add_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -8939,46 +8910,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX942-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX942-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX942-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX942-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX942-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB26_1
@@ -8989,51 +8959,52 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -9046,51 +9017,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -9103,46 +9074,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX10-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB26_1
@@ -9153,44 +9123,43 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
@@ -9201,45 +9170,44 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v1, v0, s9
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
+; GFX908-NEXT: v_mov_b32_e32 v1, v6
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB26_1
@@ -9250,46 +9218,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
@@ -9300,50 +9267,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -9352,49 +9319,49 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB26_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -9423,44 +9390,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9470,51 +9437,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -9528,50 +9492,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -9590,39 +9552,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB27_1
@@ -9635,42 +9597,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9682,43 +9644,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9730,44 +9692,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9777,43 +9739,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9825,42 +9787,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB27_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9946,14 +9908,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
; GFX942-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX942-NEXT: v_add_f32_e32 v6, v7, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX942-NEXT: v_add_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX942-NEXT: v_add3_u32 v7, v7, v6, s10
; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v6
@@ -9962,15 +9925,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX942-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX942-NEXT: v_add_f32_e32 v7, v7, v5
; GFX942-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX942-NEXT: v_add3_u32 v8, v8, v7, s10
-; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
; GFX942-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
@@ -10016,7 +9978,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2:
@@ -10038,39 +10000,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; %bb.4:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, v10, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-TRUE16-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -10081,14 +10043,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -10096,8 +10058,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_5
; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -10136,36 +10099,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; %bb.4:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v7, v6
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v7, v10, v9
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v6, v6, v9 :: v_dual_add_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-FAKE16-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -10176,14 +10139,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -10191,8 +10154,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB28_5
; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -10232,32 +10196,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v6, v7, v6
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_add_f32_e32 v7, v10, v9
-; GFX10-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX10-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v8
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v9
+; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX10-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX10-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX10-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -10268,15 +10232,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB28_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -10285,7 +10249,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_cbranch_execnz .LBB28_5
; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -10327,27 +10291,27 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
; GFX90A-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX90A-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX90A-NEXT: v_add_f32_e32 v6, v7, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s14
; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX90A-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX90A-NEXT: v_add_f32_e32 v7, v7, v5
; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s14
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
@@ -10416,33 +10380,33 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b32 s4, 0x5040100
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
; GFX908-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX908-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX908-NEXT: v_add_f32_e32 v6, v7, v6
-; GFX908-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX908-NEXT: v_add3_u32 v7, v7, v6, s14
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v8
+; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX908-NEXT: v_add3_u32 v6, v6, v5, s14
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v9
+; GFX908-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX908-NEXT: v_add3_u32 v10, v10, v6, s14
+; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX908-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX908-NEXT: v_add_f32_e32 v7, v9, v7
-; GFX908-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX908-NEXT: v_add3_u32 v9, v9, v7, s14
-; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX908-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX908-NEXT: v_perm_b32 v7, v7, v6, s15
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX908-NEXT: v_perm_b32 v6, v6, v5, s15
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -10454,21 +10418,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB28_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB28_5
; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -10508,34 +10472,34 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX8-NEXT: .LBB28_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB28_6 Depth 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX8-NEXT: v_add_f32_e32 v6, v7, v6
-; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v8
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v9
+; GFX8-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v6
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
+; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX8-NEXT: v_add_f32_e32 v7, v9, v7
-; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
-; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_alignbit_b32 v7, v7, v6, 16
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v6, v6, v5, 16
; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
; GFX8-NEXT: .LBB28_6: ; Parent Loop BB28_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -10547,21 +10511,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB28_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB28_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB28_5
; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -10771,46 +10735,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX942-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX942-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX942-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX942-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX942-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB29_1
@@ -10821,51 +10784,52 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -10878,51 +10842,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -10935,46 +10899,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX10-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB29_1
@@ -10985,44 +10948,43 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB29_1
@@ -11033,45 +10995,44 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v1, v0, s9
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
+; GFX908-NEXT: v_mov_b32_e32 v1, v6
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB29_1
@@ -11082,46 +11043,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB29_1
@@ -11132,50 +11092,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB29_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -11184,49 +11144,49 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset(ptr add
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB29_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -11255,44 +11215,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB30_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11302,51 +11262,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -11360,50 +11317,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -11422,39 +11377,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB30_1
@@ -11467,42 +11422,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11514,43 +11469,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11562,44 +11517,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11609,43 +11564,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11657,42 +11612,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset(ptr addrspace
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB30_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -11722,46 +11677,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX942-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX942-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX942-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX942-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX942-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB31_1
@@ -11772,51 +11726,52 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -11829,51 +11784,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -11886,46 +11841,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX10-NEXT: v_add_f32_e32 v1, v5, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB31_1
@@ -11936,44 +11890,43 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v1, v4, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
@@ -11984,45 +11937,44 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v1, v0, s9
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
+; GFX908-NEXT: v_mov_b32_e32 v1, v6
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB31_1
@@ -12033,46 +11985,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
@@ -12083,50 +12034,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -12135,49 +12086,49 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fadd_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB31_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v4
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -12206,44 +12157,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB32_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12253,51 +12204,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -12311,50 +12259,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -12373,39 +12319,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB32_1
@@ -12418,42 +12364,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12465,43 +12411,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB32_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12513,44 +12459,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12560,43 +12506,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12608,42 +12554,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_re
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB32_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12675,44 +12621,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB33_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12722,51 +12668,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -12780,50 +12723,48 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v3 :: v_dual_add_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -12842,39 +12783,39 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB33_1
@@ -12887,42 +12828,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_add_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12934,43 +12875,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12982,44 +12923,44 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13029,43 +12970,43 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13077,42 +13018,42 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_add_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_add_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v2, v3, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v4, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_add_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_alignbit_b32 v4, v3, v4, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB33_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13171,22 +13112,22 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v2, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB34_1
@@ -13201,12 +13142,12 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
; GFX90A-NEXT: v_add_f32_e32 v4, v5, v2
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -13229,18 +13170,18 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB34_1
@@ -13255,18 +13196,18 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB34_1
@@ -13281,18 +13222,18 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX7-NEXT: v_mov_b32_e32 v0, s20
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_add_f32_e32 v4, v5, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB34_1
@@ -13308,19 +13249,19 @@ define float @buffer_fat_ptr_system_atomic_fadd_ret_f32__offset__amdgpu_no_fine_
; GFX6-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_add_f32_e32 v3, v4, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
+; GFX6-NEXT: v_add_f32_e32 v4, v5, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB34_1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 54efdf290801c..cecc14be3c521 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -33,18 +33,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -84,18 +84,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -111,24 +111,24 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB0_1
@@ -144,19 +144,19 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
@@ -206,24 +206,24 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v1, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX942-NEXT: v_max_f32_e32 v2, v2, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -256,22 +256,22 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v1, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v2, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -282,23 +282,23 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v1, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB1_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -312,20 +312,20 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB1_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -410,13 +410,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v5
; GFX942-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX942-NEXT: v_max_f32_e32 v6, v5, v5
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v7, v9, v9
-; GFX942-NEXT: v_max_f32_e32 v8, v7, v6
+; GFX942-NEXT: v_max_f32_e32 v6, v9, v9
+; GFX942-NEXT: v_max_f32_e32 v8, v6, v5
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
; GFX942-NEXT: buffer_wbl2 sc1
@@ -527,13 +527,13 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v5
; GFX90A-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX90A-NEXT: v_max_f32_e32 v6, v5, v5
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v7, v9, v9
-; GFX90A-NEXT: v_max_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v6, v9, v9
+; GFX90A-NEXT: v_max_f32_e32 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
@@ -578,22 +578,22 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_max_f32_e32 v8, v5, v5
; GFX908-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX908-NEXT: v_max_f32_e32 v6, v5, v5
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v7, v8, v8
-; GFX908-NEXT: v_max_f32_e32 v7, v7, v6
+; GFX908-NEXT: v_max_f32_e32 v5, v7, v7
+; GFX908-NEXT: v_max_f32_e32 v6, v5, v8
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -605,21 +605,21 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB2_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -755,18 +755,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -784,27 +784,26 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, s16 :: v_dual_max_f32 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX11-NEXT: v_max_f32_e32 v3, v1, v0
-; GFX11-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX11-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -816,26 +815,26 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f32_e32 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX10-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
@@ -846,18 +845,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -873,24 +872,24 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
@@ -906,19 +905,19 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
@@ -934,19 +933,19 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX7-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB3_1
@@ -963,20 +962,20 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX6-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB3_1
@@ -1007,18 +1006,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1058,18 +1057,18 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1085,24 +1084,24 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
@@ -1118,19 +1117,19 @@ define float @buffer_fat_ptr_agent_atomic_fmax_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
@@ -1173,26 +1172,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1215,28 +1214,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1268,29 +1266,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1300,29 +1296,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1361,25 +1355,24 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[6:7], v[2:3]
-; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1403,27 +1396,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_max_f64 v[2:3], v[6:7], v[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX11-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_mov_b32_e32 v6, v2
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1456,26 +1447,26 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[2:3], v[6:7], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v9, v5
-; GFX908-NEXT: v_mov_b32_e32 v8, v4
-; GFX908-NEXT: v_mov_b32_e32 v7, v3
-; GFX908-NEXT: v_mov_b32_e32 v6, v2
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v6
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v7
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB6_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1486,26 +1477,26 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f64__offset__amdgpu_no_fine_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[2:3], v[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v8, v4
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v6
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v7
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1564,19 +1555,19 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[5:6], v[5:6]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[13:14], v[13:14]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[2:3], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
@@ -1661,18 +1652,18 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX11-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_max_f64 v[11:12], v[2:3], v[0:1]
+; GFX11-NEXT: v_max_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
@@ -1785,15 +1776,15 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX908-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_max_f64 v[11:12], v[2:3], v[0:1]
+; GFX908-NEXT: v_max_f64 v[11:12], v[0:1], v[5:6]
; GFX908-NEXT: v_mov_b32_e32 v2, v13
; GFX908-NEXT: v_mov_b32_e32 v3, v14
; GFX908-NEXT: v_mov_b32_e32 v0, v11
@@ -1849,15 +1840,15 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__waterfall__amdg
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX8-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_max_f64 v[11:12], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[11:12], v[0:1], v[5:6]
; GFX8-NEXT: v_mov_b32_e32 v2, v13
; GFX8-NEXT: v_mov_b32_e32 v3, v14
; GFX8-NEXT: v_mov_b32_e32 v0, v11
@@ -1957,26 +1948,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1999,28 +1990,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2032,30 +2022,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v8, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
+; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
+; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v2, v8
-; GFX10-NEXT: v_mov_b32_e32 v3, v9
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
-; GFX10-NEXT: v_mov_b32_e32 v1, v7
-; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX10-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v3
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v3, v5
+; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT: v_mov_b32_e32 v9, v1
-; GFX10-NEXT: v_mov_b32_e32 v8, v0
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
@@ -2066,26 +2054,24 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX90A-NEXT: v_mov_b32_e32 v5, v1
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX90A-NEXT: v_mov_b32_e32 v8, s20
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX90A-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX90A-NEXT: v_mov_b32_e32 v10, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX90A-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2095,29 +2081,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2127,29 +2111,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2159,29 +2141,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s20
-; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v8, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
-; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v2, v8
-; GFX7-NEXT: v_mov_b32_e32 v0, v6
-; GFX7-NEXT: v_mov_b32_e32 v1, v7
-; GFX7-NEXT: v_mov_b32_e32 v3, v9
-; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX7-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, v4
+; GFX7-NEXT: v_mov_b32_e32 v3, v5
+; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2191,31 +2171,28 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remot
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_mov_b32_e32 v0, s20
-; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
+; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v8, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
-; GFX6-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, v8
-; GFX6-NEXT: v_mov_b32_e32 v0, v6
-; GFX6-NEXT: v_mov_b32_e32 v1, v7
-; GFX6-NEXT: v_mov_b32_e32 v3, v9
-; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX6-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v9, v1
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v8, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2235,26 +2212,26 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2277,28 +2254,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2330,29 +2306,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2362,29 +2336,27 @@ define double @buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2427,46 +2399,46 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2477,355 +2449,346 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v4, s4
+; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f16_e32 v5, v0, v0
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v0, v0
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
+; GFX10-NEXT: v_max_f16_e32 v5, v0, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v5, v0, v0
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v5, v0, v0
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v5, v0, v0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v4, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_max_f32_e32 v0, v0, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -2835,37 +2798,36 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_max_f32_e32 v0, v0, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -2875,7 +2837,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: s_cbranch_execnz .LBB10_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
@@ -2892,45 +2854,45 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2941,78 +2903,77 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f16_e32 v3, v0, v0
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3023,159 +2984,154 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
+; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v2, s4
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v3, v0, v0
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3186,32 +3142,31 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3222,33 +3177,32 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v4, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v3
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3259,34 +3213,33 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_max_f32_e32 v0, v0, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3297,35 +3250,34 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_f16__offset__amdgpu_no_fine_
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_max_f32_e32 v0, v0, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB11_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3345,17 +3297,17 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3366,31 +3318,31 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.h, v6.l, v6.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v6.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.h, v5.h, v6.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v4.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3402,14 +3354,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3417,7 +3369,7 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -3427,17 +3379,17 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3448,31 +3400,31 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v8, v5, v5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v8
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v10
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX12-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3484,14 +3436,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3499,19 +3451,19 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX942-NEXT: v_and_b32_e32 v9, -4, v4
; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v8, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v6, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v11, v6
+; GFX942-NEXT: v_lshlrev_b32_e64 v4, v8, s0
+; GFX942-NEXT: v_not_b32_e32 v10, v4
; GFX942-NEXT: s_mov_b64 s[2:3], exec
; GFX942-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
@@ -3523,24 +3475,24 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX942-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_max_f16_e32 v11, v5, v5
; GFX942-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB12_4 Depth 2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX942-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX942-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX942-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX942-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX942-NEXT: v_lshrrev_b32_e32 v4, v8, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v11
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, v8, v4
+; GFX942-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_mov_b64_e32 v[8:9], v[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
@@ -3555,37 +3507,37 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v7, v8
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB12_3
; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3594,30 +3546,30 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.h, v6.l, v6.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v6.l, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.h, v5.h, v6.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v4.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3628,14 +3580,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -3644,23 +3596,23 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3669,30 +3621,30 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v6, v6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v10
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX11-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3703,14 +3655,14 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -3719,21 +3671,21 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX10-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v11, v7
+; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -3742,27 +3694,27 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX10-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX10-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX10-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX10-NEXT: v_mov_b32_e32 v9, v7
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX10-NEXT: v_max_f16_e32 v4, v4, v10
+; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, v5
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -3773,15 +3725,15 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -3790,19 +3742,19 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX90A-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX90A-NEXT: v_and_b32_e32 v9, -4, v4
; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v8, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX90A-NEXT: v_not_b32_e32 v11, v6
+; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
+; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
; GFX90A-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
@@ -3814,24 +3766,24 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_max_f16_e32 v11, v5, v5
; GFX90A-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB12_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX90A-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX90A-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX90A-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX90A-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v8, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v11
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v8, v4
+; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
@@ -3843,33 +3795,33 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v7, v8
+; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX908-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX908-NEXT: v_and_b32_e32 v8, -4, v4
; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v7, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX908-NEXT: v_not_b32_e32 v11, v6
+; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
+; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
; GFX908-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -3881,25 +3833,25 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_max_f16_e32 v10, v5, v5
; GFX908-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB12_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX908-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX908-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX908-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX908-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX908-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX908-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX908-NEXT: v_max_f16_e32 v4, v4, v10
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX908-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_mov_b32_e32 v9, v7
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v4, v5
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -3911,33 +3863,33 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB12_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
-; GFX8-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX8-NEXT: v_and_b32_e32 v8, -4, v4
; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX8-NEXT: v_not_b32_e32 v11, v6
+; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
+; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -3949,26 +3901,26 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_max_f16_e32 v10, v5, v5
; GFX8-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB12_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX8-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX8-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX8-NEXT: v_max_f16_e32 v6, v6, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX8-NEXT: v_and_b32_e32 v8, v7, v11
-; GFX8-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX8-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX8-NEXT: v_max_f16_e32 v4, v4, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX8-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v4
; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_mov_b32_e32 v9, v7
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -3980,21 +3932,21 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB12_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -4151,27 +4103,27 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -4181,56 +4133,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v4, s4
+; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v5, s6
-; GFX942-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v5
; GFX942-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX942-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4240,32 +4193,33 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -4275,97 +4229,95 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v5
; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v5
; GFX90A-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4375,39 +4327,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v5
; GFX908-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX908-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX908-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX908-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4417,41 +4368,40 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v5
; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, s8, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4461,39 +4411,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_max_f32_e32 v0, v0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4503,40 +4452,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_max_f32_e32 v0, v0, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4546,7 +4494,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: s_cbranch_execnz .LBB13_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -4563,85 +4511,86 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v3, s6
-; GFX942-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v3
; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX942-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -4657,123 +4606,122 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v3
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, v1
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v2, s4
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -4789,37 +4737,36 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
-; GFX908-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v0, s9
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX908-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX908-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v1
-; GFX908-NEXT: v_mov_b32_e32 v3, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4830,39 +4777,38 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f32_e32 v4, v4, v2
-; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4873,37 +4819,36 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_max_f32_e32 v0, v0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4914,38 +4859,37 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_max_f32_e32 v0, v0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB14_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5660,28 +5604,28 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v1, v1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v2, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v1, v4, v4
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v3, v1, v0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v4, v0, v2
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5697,31 +5641,31 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v1, v4, v4
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v3, v1, v0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v4, v0, v2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5733,25 +5677,25 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_pk_max_f16 v1, v2, v2
; GFX942-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_pk_max_f16 v4, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -5767,29 +5711,29 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v1, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v4, v4
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v3, v1, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5801,32 +5745,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v1, v4, v4
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v3, v1, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5838,32 +5782,31 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_max_f16 v1, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX10-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB16_1
@@ -5874,23 +5817,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_pk_max_f16 v1, v2, v2
; GFX90A-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_pk_max_f16 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5906,29 +5848,28 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v0, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX908-NEXT: v_pk_max_f16 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB16_1
@@ -5939,33 +5880,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v6, v0
-; GFX8-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB16_1
@@ -5984,27 +5924,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6027,27 +5967,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6072,28 +6012,28 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v1, v0, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v3, v2, v2
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v1, v3, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6113,27 +6053,27 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v1, v0, v0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v3, v2, v2
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v1, v3, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6149,26 +6089,27 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_pk_max_f16 v2, v2, v1
+; GFX942-NEXT: v_pk_max_f16 v0, v1, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6178,29 +6119,29 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v1, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v3, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6216,28 +6157,28 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v1, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v1, v3, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6255,25 +6196,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_max_f16 v1, v3, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB17_1
@@ -6288,24 +6229,24 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v2, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6319,25 +6260,25 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v1, v3, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX908-NEXT: v_pk_max_f16 v0, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6349,31 +6290,31 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v4, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v3, v5, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6391,27 +6332,27 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6431,27 +6372,27 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_max_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6486,7 +6427,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-TRUE16-NEXT: ; %bb.2:
@@ -6511,21 +6452,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: ; %bb.4:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v6, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v7, v8, v8
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v5, v7, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v7, v7, v6
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v6, v5, v8
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6537,14 +6478,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6552,7 +6493,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6600,21 +6541,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: ; %bb.4:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v6, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v7, v8, v8
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v5, v7, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v7, v7, v6
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v6, v5, v8
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6626,14 +6567,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6641,7 +6582,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6683,13 +6624,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_pk_max_f16 v5, v5, v5
; GFX942-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX942-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX942-NEXT: v_pk_max_f16 v7, v9, v9
+; GFX942-NEXT: v_pk_max_f16 v6, v9, v9
; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_pk_max_f16 v8, v7, v6
+; GFX942-NEXT: v_pk_max_f16 v8, v6, v5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
@@ -6736,7 +6677,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
@@ -6758,20 +6699,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: ; %bb.4:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v6, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v5, v7, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v7, v7, v6
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v6, v5, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6782,14 +6723,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6798,7 +6739,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6837,20 +6778,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: ; %bb.4:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v6, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v5, v7, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v7, v7, v6
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v6, v5, v8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6861,14 +6802,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6877,7 +6818,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6917,18 +6858,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX10-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX10-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_pk_max_f16 v7, v7, v6
+; GFX10-NEXT: v_pk_max_f16 v6, v5, v8
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -6939,15 +6880,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -6956,7 +6897,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_cbranch_execnz .LBB18_5
; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6998,12 +6939,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_pk_max_f16 v5, v5, v5
; GFX90A-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX90A-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v7, v9, v9
-; GFX90A-NEXT: v_pk_max_f16 v8, v7, v6
+; GFX90A-NEXT: v_pk_max_f16 v6, v9, v9
+; GFX90A-NEXT: v_pk_max_f16 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
@@ -7071,17 +7012,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b32 s4, 0x5040100
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
; GFX908-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX908-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX908-NEXT: v_pk_max_f16 v7, v8, v8
-; GFX908-NEXT: v_pk_max_f16 v7, v7, v6
+; GFX908-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX908-NEXT: v_pk_max_f16 v6, v5, v8
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -7093,21 +7034,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB18_5
; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7147,21 +7088,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
; GFX8-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX8-NEXT: v_max_f16_sdwa v6, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v7, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v7, v5, v5
-; GFX8-NEXT: v_max_f16_e32 v9, v8, v8
-; GFX8-NEXT: v_max_f16_e32 v7, v9, v7
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v6
+; GFX8-NEXT: v_max_f16_sdwa v5, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v7, v7
+; GFX8-NEXT: v_max_f16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v9
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
; GFX8-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -7173,21 +7114,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB18_5
; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7380,49 +7321,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v1, v0
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v1, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7438,50 +7381,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v1, v5, v1
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v1, v1, v3
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -7493,46 +7436,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v1, v4, v1
-; GFX942-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX942-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX942-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX942-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB19_1
@@ -7543,51 +7485,52 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7600,51 +7543,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v1, v5, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -7657,46 +7600,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX10-NEXT: v_max_f32_e32 v1, v5, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB19_1
@@ -7707,44 +7649,43 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v1, v4, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
@@ -7755,45 +7696,44 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v1, v0, s9
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
+; GFX908-NEXT: v_mov_b32_e32 v1, v6
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB19_1
@@ -7804,46 +7744,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_max_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB19_1
@@ -7854,50 +7793,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_max_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -7906,49 +7845,49 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_max_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_max_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -7965,49 +7904,47 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v1, v3, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v5, v5, v3 :: v_dual_max_num_f32 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8024,49 +7961,47 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v1, v3, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
+; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v5, v3 :: v_dual_max_num_f32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -8080,44 +8015,44 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_max_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB20_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8127,51 +8062,48 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_max_f32 v5, v5, v3 :: v_dual_max_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8185,50 +8117,48 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v5, v3 :: v_dual_max_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -8247,39 +8177,39 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB20_1
@@ -8292,42 +8222,42 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_max_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8339,43 +8269,43 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8387,44 +8317,44 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8434,43 +8364,43 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_max_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_max_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8482,42 +8412,42 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_max_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_max_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_max_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_max_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8550,7 +8480,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
@@ -8575,42 +8505,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: ; %bb.4:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v7, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v7, v10, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v6, v6, v9 :: v_dual_max_num_f32 v5, v5, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8622,14 +8551,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8637,7 +8566,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8685,39 +8614,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: ; %bb.4:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v7, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v7, v10, v9
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX12-FAKE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v6, v6, v9 :: v_dual_max_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8729,14 +8657,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8744,7 +8672,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8786,14 +8714,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
; GFX942-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX942-NEXT: v_max_f32_e32 v6, v7, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX942-NEXT: v_add3_u32 v7, v7, v6, s10
; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v6
@@ -8802,15 +8731,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX942-NEXT: v_max_f32_e32 v7, v8, v7
+; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX942-NEXT: v_max_f32_e32 v7, v7, v5
; GFX942-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX942-NEXT: v_add3_u32 v8, v8, v7, s10
-; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
; GFX942-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
@@ -8856,7 +8784,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
@@ -8878,39 +8806,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; %bb.4:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v7, v10, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8921,14 +8849,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8936,8 +8864,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8976,36 +8905,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; %bb.4:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v7, v6
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v7, v10, v9
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-FAKE16-NEXT: v_dual_max_f32 v6, v6, v9 :: v_dual_max_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -9016,14 +8945,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -9031,8 +8960,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9072,32 +9002,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v6, v7, v6
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_max_f32_e32 v7, v10, v9
-; GFX10-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX10-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v8
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v9
+; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX10-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX10-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX10-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -9108,15 +9038,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -9125,7 +9055,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_cbranch_execnz .LBB21_5
; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9167,27 +9097,27 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
; GFX90A-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX90A-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX90A-NEXT: v_max_f32_e32 v6, v7, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s14
; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX90A-NEXT: v_max_f32_e32 v7, v8, v7
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX90A-NEXT: v_max_f32_e32 v7, v7, v5
; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s14
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
@@ -9256,33 +9186,33 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b32 s4, 0x5040100
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
; GFX908-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX908-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX908-NEXT: v_max_f32_e32 v6, v7, v6
-; GFX908-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX908-NEXT: v_add3_u32 v7, v7, v6, s14
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v8
+; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX908-NEXT: v_add3_u32 v6, v6, v5, s14
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v9
+; GFX908-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX908-NEXT: v_add3_u32 v10, v10, v6, s14
+; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX908-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX908-NEXT: v_max_f32_e32 v7, v9, v7
-; GFX908-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX908-NEXT: v_add3_u32 v9, v9, v7, s14
-; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX908-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX908-NEXT: v_perm_b32 v7, v7, v6, s15
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX908-NEXT: v_perm_b32 v6, v6, v5, s15
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -9294,21 +9224,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB21_5
; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9348,34 +9278,34 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX8-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX8-NEXT: v_max_f32_e32 v6, v7, v6
-; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v8
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v9
+; GFX8-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v6
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
+; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX8-NEXT: v_max_f32_e32 v7, v9, v7
-; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
-; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_alignbit_b32 v7, v7, v6, 16
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v6, v6, v5, 16
; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
; GFX8-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -9387,21 +9317,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB21_5
; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9617,18 +9547,18 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX942-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_max_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9668,18 +9598,18 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_max_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9698,24 +9628,24 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_1
@@ -9731,19 +9661,19 @@ define float @buffer_fat_ptr_system_atomic_fmax_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_max_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index f4512c9ac18ce..7859ac4841a10 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -33,18 +33,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_min_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -84,18 +84,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -111,24 +111,24 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB0_1
@@ -144,19 +144,19 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB0_1
@@ -206,24 +206,24 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v1, v0, v0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX942-NEXT: v_min_f32_e32 v2, v2, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -256,22 +256,22 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v1, v0, v0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v2, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -282,23 +282,23 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v0, v0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v1, v0, v0
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX908-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB1_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -312,20 +312,20 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v1
; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB1_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -410,13 +410,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v5
; GFX942-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX942-NEXT: v_max_f32_e32 v6, v5, v5
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v7, v9, v9
-; GFX942-NEXT: v_min_f32_e32 v8, v7, v6
+; GFX942-NEXT: v_max_f32_e32 v6, v9, v9
+; GFX942-NEXT: v_min_f32_e32 v8, v6, v5
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
; GFX942-NEXT: buffer_wbl2 sc1
@@ -527,13 +527,13 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v5
; GFX90A-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX90A-NEXT: v_max_f32_e32 v6, v5, v5
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v7, v9, v9
-; GFX90A-NEXT: v_min_f32_e32 v8, v7, v6
+; GFX90A-NEXT: v_max_f32_e32 v6, v9, v9
+; GFX90A-NEXT: v_min_f32_e32 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
@@ -578,22 +578,22 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v8, v4, s[8:11], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_dword v7, v4, s[8:11], 0 offen offset:1024
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_max_f32_e32 v8, v5, v5
; GFX908-NEXT: .LBB2_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB2_4 Depth 2
-; GFX908-NEXT: v_max_f32_e32 v6, v5, v5
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v7, v8, v8
-; GFX908-NEXT: v_min_f32_e32 v7, v7, v6
+; GFX908-NEXT: v_max_f32_e32 v5, v7, v7
+; GFX908-NEXT: v_min_f32_e32 v6, v5, v8
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB2_4: ; Parent Loop BB2_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -605,21 +605,21 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgp
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB2_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB2_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -755,18 +755,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_min_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -784,27 +784,26 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, s16 :: v_dual_max_f32 v2, v1, v1
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX11-NEXT: v_min_f32_e32 v3, v1, v0
-; GFX11-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX11-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -816,26 +815,26 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, v0
; GFX10-NEXT: v_mov_b32_e32 v0, s20
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f32_e32 v2, v1, v1
; GFX10-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX10-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_max_f32_e32 v0, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f32_e32 v1, v4, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX10-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB3_1
@@ -846,18 +845,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -873,24 +872,24 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
@@ -906,19 +905,19 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB3_1
@@ -934,19 +933,19 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX7-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX7-NEXT: v_mov_b32_e32 v3, s20
; GFX7-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX7-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s20
-; GFX7-NEXT: v_mov_b32_e32 v0, v3
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v0
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX7-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB3_1
@@ -963,20 +962,20 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_remote
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, s6
; GFX6-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX6-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_mov_b32_e32 v0, v3
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX6-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB3_1
@@ -1007,18 +1006,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_min_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -1058,18 +1057,18 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -1085,24 +1084,24 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
@@ -1118,19 +1117,19 @@ define float @buffer_fat_ptr_agent_atomic_fmin_ret_f32__offset__amdgpu_no_fine_g
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB4_1
@@ -1173,26 +1172,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1215,28 +1214,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1268,29 +1266,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1300,29 +1296,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1361,25 +1355,24 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v6, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[0:1]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[6:7], v[2:3]
-; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX12-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_mov_b32_e32 v6, v2
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX12-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX12-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1403,27 +1396,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v6, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_min_f64 v[2:3], v[6:7], v[2:3]
-; GFX11-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, v5
-; GFX11-NEXT: v_dual_mov_b32 v8, v4 :: v_dual_mov_b32 v7, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_mov_b32_e32 v6, v2
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[6:9], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v10, v3 :: v_dual_mov_b32 v9, v2
+; GFX11-NEXT: v_dual_mov_b32 v8, v1 :: v_dual_mov_b32 v7, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[7:10], v6, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[4:5]
-; GFX11-NEXT: v_dual_mov_b32 v4, v6 :: v_dual_mov_b32 v5, v7
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[7:8], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v2, v7 :: v_dual_mov_b32 v3, v8
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1456,26 +1447,26 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v6, s20
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[2:3], v[6:7], v[2:3]
-; GFX908-NEXT: v_mov_b32_e32 v9, v5
-; GFX908-NEXT: v_mov_b32_e32 v8, v4
-; GFX908-NEXT: v_mov_b32_e32 v7, v3
-; GFX908-NEXT: v_mov_b32_e32 v6, v2
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v10, v3
+; GFX908-NEXT: v_mov_b32_e32 v9, v2
+; GFX908-NEXT: v_mov_b32_e32 v8, v1
+; GFX908-NEXT: v_mov_b32_e32 v7, v0
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v6
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v2, v7
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v5, v7
+; GFX908-NEXT: v_mov_b32_e32 v3, v8
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB6_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1486,26 +1477,26 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f64__offset__amdgpu_no_fine_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: buffer_load_dwordx2 v[2:3], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s20
; GFX8-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[2:3], v[0:1], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[6:7], v[4:5], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[2:3], v[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v9, v5
-; GFX8-NEXT: v_mov_b32_e32 v8, v4
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v6, v2
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[6:9], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v10, v3
+; GFX8-NEXT: v_mov_b32_e32 v9, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[7:10], v6, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v6
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[7:8], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v2, v7
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v5, v7
+; GFX8-NEXT: v_mov_b32_e32 v3, v8
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB6_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1564,19 +1555,19 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX12-NEXT: s_cbranch_execnz .LBB7_1
; GFX12-NEXT: ; %bb.2:
; GFX12-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-NEXT: v_max_num_f64_e32 v[5:6], v[5:6], v[5:6]
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Loop Header: Depth=1
; GFX12-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[5:6], v[5:6]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[13:14], v[13:14]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[13:14], v[13:14]
; GFX12-NEXT: s_mov_b32 s5, exec_lo
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_mov_b32 s6, s5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[2:3], v[0:1]
+; GFX12-NEXT: v_min_num_f64_e32 v[11:12], v[0:1], v[5:6]
; GFX12-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX12-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX12-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
@@ -1661,18 +1652,18 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2:
; GFX11-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Loop Header: Depth=1
; GFX11-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX11-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX11-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX11-NEXT: s_mov_b32 s5, exec_lo
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_mov_b32 s6, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_min_f64 v[11:12], v[2:3], v[0:1]
+; GFX11-NEXT: v_min_f64 v[11:12], v[0:1], v[5:6]
; GFX11-NEXT: v_dual_mov_b32 v2, v13 :: v_dual_mov_b32 v3, v14
; GFX11-NEXT: v_dual_mov_b32 v0, v11 :: v_dual_mov_b32 v1, v12
; GFX11-NEXT: .LBB7_4: ; Parent Loop BB7_3 Depth=1
@@ -1785,15 +1776,15 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
+; GFX908-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX908-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX908-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_min_f64 v[11:12], v[2:3], v[0:1]
+; GFX908-NEXT: v_min_f64 v[11:12], v[0:1], v[5:6]
; GFX908-NEXT: v_mov_b32_e32 v2, v13
; GFX908-NEXT: v_mov_b32_e32 v3, v14
; GFX908-NEXT: v_mov_b32_e32 v0, v11
@@ -1849,15 +1840,15 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__waterfall__amdg
; GFX8-NEXT: s_cbranch_execnz .LBB7_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
+; GFX8-NEXT: v_max_f64 v[5:6], v[5:6], v[5:6]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB7_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB7_4 Depth 2
-; GFX8-NEXT: v_max_f64 v[0:1], v[5:6], v[5:6]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[13:14], v[13:14]
+; GFX8-NEXT: v_max_f64 v[0:1], v[13:14], v[13:14]
; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_min_f64 v[11:12], v[2:3], v[0:1]
+; GFX8-NEXT: v_min_f64 v[11:12], v[0:1], v[5:6]
; GFX8-NEXT: v_mov_b32_e32 v2, v13
; GFX8-NEXT: v_mov_b32_e32 v3, v14
; GFX8-NEXT: v_mov_b32_e32 v0, v11
@@ -1957,26 +1948,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -1999,28 +1990,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2032,30 +2022,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
-; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
+; GFX10-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX10-NEXT: v_mov_b32_e32 v8, s20
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
+; GFX10-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX10-NEXT: v_mov_b32_e32 v10, s20
+; GFX10-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX10-NEXT: v_mov_b32_e32 v2, v8
-; GFX10-NEXT: v_mov_b32_e32 v3, v9
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
-; GFX10-NEXT: v_mov_b32_e32 v1, v7
-; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX10-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v3
+; GFX10-NEXT: v_mov_b32_e32 v2, v4
+; GFX10-NEXT: v_mov_b32_e32 v3, v5
+; GFX10-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX10-NEXT: v_mov_b32_e32 v9, v1
-; GFX10-NEXT: v_mov_b32_e32 v8, v0
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB8_1
@@ -2066,26 +2054,24 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v4, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX90A-NEXT: v_mov_b32_e32 v5, v1
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
+; GFX90A-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX90A-NEXT: v_mov_b32_e32 v8, s20
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX90A-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX90A-NEXT: v_mov_b32_e32 v10, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[8:9], v[8:9] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX90A-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX90A-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2095,29 +2081,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB8_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2127,29 +2111,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2159,29 +2141,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s20
-; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v8, s20
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v10, s20
-; GFX7-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v2, v8
-; GFX7-NEXT: v_mov_b32_e32 v0, v6
-; GFX7-NEXT: v_mov_b32_e32 v1, v7
-; GFX7-NEXT: v_mov_b32_e32 v3, v9
-; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX7-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, v4
+; GFX7-NEXT: v_mov_b32_e32 v3, v5
+; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2191,31 +2171,28 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remot
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_remote_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v4, v0
-; GFX6-NEXT: v_mov_b32_e32 v0, s20
-; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v2, s20
+; GFX6-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
; GFX6-NEXT: s_add_i32 s6, s20, 0x800
+; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v8, s6
; GFX6-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v10, s6
-; GFX6-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, v8
-; GFX6-NEXT: v_mov_b32_e32 v0, v6
-; GFX6-NEXT: v_mov_b32_e32 v1, v7
-; GFX6-NEXT: v_mov_b32_e32 v3, v9
-; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen glc
+; GFX6-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX6-NEXT: v_mov_b32_e32 v0, v2
+; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, v4
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v9, v1
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v8, v0
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB8_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2235,26 +2212,26 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-NEXT: v_mov_b32_e32 v2, s16
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v8, s16
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], null offen offset:2048
+; GFX12-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null offen offset:2048
; GFX12-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v10, s16
-; GFX12-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX12-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[6:7]
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-NEXT: v_mov_b32_e32 v3, v5
+; GFX12-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], null offen offset:2048 th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2277,28 +2254,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-NEXT: v_mov_b32_e32 v2, s16
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
+; GFX11-NEXT: v_mov_b32_e32 v8, s16
; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: buffer_load_b64 v[8:9], v0, s[0:3], 0 offen offset:2048
-; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], 0 offen offset:2048
; GFX11-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX11-NEXT: v_mov_b32_e32 v10, s16
-; GFX11-NEXT: v_dual_mov_b32 v2, v8 :: v_dual_mov_b32 v3, v9
-; GFX11-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
-; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v10, s[0:3], 0 offen offset:2048 glc
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-NEXT: v_mov_b32_e32 v3, v5
+; GFX11-NEXT: buffer_atomic_cmpswap_b64 v[0:3], v8, s[0:3], 0 offen offset:2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[8:9]
-; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -2330,29 +2306,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
+; GFX908-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_mov_b32_e32 v8, s20
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v10, s20
-; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX908-NEXT: v_mov_b32_e32 v2, v8
-; GFX908-NEXT: v_mov_b32_e32 v3, v9
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
-; GFX908-NEXT: v_mov_b32_e32 v1, v7
-; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v2, v4
+; GFX908-NEXT: v_mov_b32_e32 v3, v5
+; GFX908-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB9_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2362,29 +2336,27 @@ define double @buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f64__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
-; GFX8-NEXT: buffer_load_dwordx2 v[8:9], v0, s[16:19], 0 offen offset:2048
-; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_dwordx2 v[4:5], v2, s[16:19], 0 offen offset:2048
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v8, s20
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v10, s20
-; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v2, v8
-; GFX8-NEXT: v_mov_b32_e32 v3, v9
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
-; GFX8-NEXT: v_mov_b32_e32 v1, v7
-; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v10, s[16:19], 0 offen offset:2048 glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[0:1], v[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, v5
+; GFX8-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v8, s[16:19], 0 offen offset:2048 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2427,46 +2399,46 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2477,355 +2449,346 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v4, s4
+; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f16_e32 v5, v0, v0
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX942-NEXT: v_min_f16_e32 v1, v1, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX942-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v5, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v0, v0
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v1, v1, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
+; GFX10-NEXT: v_max_f16_e32 v5, v0, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_mov_b32_e32 v2, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v2
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB10_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v3
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v5, v0, v0
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX90A-NEXT: v_min_f16_e32 v1, v1, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX90A-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v5, v0, v0
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX908-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX908-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB10_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v5, v0, v0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v4, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-NEXT: v_min_f16_e32 v0, v0, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_min_f32_e32 v0, v0, v5
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -2835,37 +2798,36 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_min_f32_e32 v0, v0, v5
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -2875,7 +2837,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__amdgpu_no_fine_gr
; GFX6-NEXT: s_cbranch_execnz .LBB10_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr half, ptr addrspace(7) %ptr, i32 256
@@ -2892,45 +2854,45 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX12-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v1.l, v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v1.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
@@ -2941,78 +2903,77 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_and_b32 s5, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX12-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v0, v0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v1, v1, v1
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v1, v1, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v0, v0, v0
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX12-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v3, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f16_e32 v3, v0, v0
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX942-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX942-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX942-NEXT: v_min_f16_e32 v1, v1, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX942-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s6
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX942-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX942-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3023,159 +2984,154 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_not_b32 s7, s6
-; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, 0
-; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v2, v3, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
; GFX11-TRUE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, s4, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v1.l, v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, s4, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v1, v2, s6, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-TRUE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s16, 3
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_not_b32 s7, s6
-; GFX11-FAKE16-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v0, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v1, v1, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, s5, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB11_1
; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
+; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, s5, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX10-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v1, v2, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX10-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB11_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v3, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v2, s4
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v3, v0, v0
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v1, s7, v3
-; GFX90A-NEXT: v_max_f16_e32 v2, v0, v0
-; GFX90A-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX90A-NEXT: v_min_f16_e32 v1, v1, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX90A-NEXT: v_and_or_b32 v2, v3, s8, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s6
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX90A-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX90A-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3186,32 +3142,31 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX908-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX908-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX908-NEXT: v_and_or_b32 v1, v2, s8, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX908-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX908-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB11_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3222,33 +3177,32 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s7, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
-; GFX8-NEXT: v_min_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_and_b32_e32 v4, s8, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s7, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v1
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v0
+; GFX8-NEXT: v_min_f16_e32 v0, v0, v3
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3259,34 +3213,33 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_min_f32_e32 v0, v0, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3297,35 +3250,34 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_f16__offset__amdgpu_no_fine_
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
; GFX6-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_min_f32_e32 v0, v0, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB11_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3345,17 +3297,17 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-TRUE16-NEXT: s_mov_b32 s4, exec_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX12-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3366,31 +3318,31 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], null offen
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-TRUE16-NEXT: ; %bb.2:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.h, v6.l, v6.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v6.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v4.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v5.h, v5.h, v6.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v4.h, v4.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX12-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3402,14 +3354,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3417,7 +3369,7 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -3427,17 +3379,17 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX12-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s4, exec_lo
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s5, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v11, v7
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX12-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3448,31 +3400,31 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], null offen
+; GFX12-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], null offen
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX12-FAKE16-NEXT: ; %bb.2:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v10, v5, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v8, v5, v5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v6, v6
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v4, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v6, v6, v8
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v4, v4, v10
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX12-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GFX12-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3484,14 +3436,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX12-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3499,19 +3451,19 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX12-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX942-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX942-NEXT: v_and_b32_e32 v9, -4, v4
; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v8, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v6, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v11, v6
+; GFX942-NEXT: v_lshlrev_b32_e64 v4, v8, s0
+; GFX942-NEXT: v_not_b32_e32 v10, v4
; GFX942-NEXT: s_mov_b64 s[2:3], exec
; GFX942-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_readfirstlane_b32 s4, v0
@@ -3523,24 +3475,24 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: v_cmp_eq_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
-; GFX942-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX942-NEXT: buffer_load_dword v7, v9, s[4:7], 0 offen
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2:
; GFX942-NEXT: s_mov_b64 exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_max_f16_e32 v11, v5, v5
; GFX942-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB12_4 Depth 2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX942-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX942-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX942-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX942-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX942-NEXT: v_lshrrev_b32_e32 v4, v8, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX942-NEXT: v_min_f16_e32 v4, v4, v11
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, v8, v4
+; GFX942-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_mov_b64_e32 v[8:9], v[6:7]
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[6:7]
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
@@ -3555,37 +3507,37 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX942-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], s[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[4:7], 0 offen sc0
; GFX942-NEXT: s_xor_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_4
; GFX942-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX942-NEXT: s_mov_b64 exec, s[8:9]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v7, v8
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB12_3
; GFX942-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 3, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v9, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v11, v6
; GFX11-TRUE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3594,30 +3546,30 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: buffer_load_b32 v6, v10, s[0:3], 0 offen
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-TRUE16-NEXT: ; %bb.2:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v9, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.h, v6.l, v6.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v6.l, v5.l, v5.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v5.h, v5.h, v6.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v6, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v4.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v4.h, v4.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v9, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v11, v5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v6 :: v_dual_mov_b32 v7, v5
; GFX11-TRUE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3628,14 +3580,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[7:8], v10, s[0:3], 0 offen glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-TRUE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v7, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v7
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -3644,23 +3596,23 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-TRUE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v9, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 3, v6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v11, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v9, v6
; GFX11-FAKE16-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
@@ -3669,30 +3621,30 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-FAKE16-NEXT: buffer_load_b32 v7, v10, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: buffer_load_b32 v6, v8, s[0:3], 0 offen
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_1
; GFX11-FAKE16-NEXT: ; %bb.2:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v10, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB12_4 Depth 2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v6, v6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v4, v4, v10
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX11-FAKE16-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v9, v7 :: v_dual_mov_b32 v8, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v5 :: v_dual_mov_b32 v5, v6
; GFX11-FAKE16-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -3703,14 +3655,14 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[8:9], v10, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v8, s[0:3], 0 offen glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_4
; GFX11-FAKE16-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v4
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -3719,21 +3671,21 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB12_3
; GFX11-FAKE16-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_nc_u32_e32 v6, 0x200, v4
+; GFX10-NEXT: v_add_nc_u32_e32 v4, 0x200, v4
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_and_b32_e32 v4, 3, v6
-; GFX10-NEXT: v_and_b32_e32 v10, -4, v6
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v11, v7
+; GFX10-NEXT: v_and_b32_e32 v6, 3, v4
+; GFX10-NEXT: v_and_b32_e32 v8, -4, v4
+; GFX10-NEXT: v_lshlrev_b32_e32 v7, 3, v6
+; GFX10-NEXT: v_lshlrev_b32_e64 v6, v7, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v9, v6
; GFX10-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
; GFX10-NEXT: v_readfirstlane_b32 s5, v1
@@ -3742,27 +3694,27 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
-; GFX10-NEXT: buffer_load_dword v7, v10, s[4:7], 0 offen
+; GFX10-NEXT: buffer_load_dword v6, v8, s[4:7], 0 offen
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_1
; GFX10-NEXT: ; %bb.2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
+; GFX10-NEXT: v_max_f16_e32 v10, v5, v5
; GFX10-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB12_4 Depth 2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX10-NEXT: v_max_f16_e32 v8, v5, v5
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, v7, v6
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX10-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX10-NEXT: v_lshlrev_b32_sdwa v6, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v6, v7, v11, v6
-; GFX10-NEXT: v_mov_b32_e32 v9, v7
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX10-NEXT: v_min_f16_e32 v4, v4, v10
+; GFX10-NEXT: v_lshlrev_b32_sdwa v4, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v5, v6, v9, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, v5
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -3773,15 +3725,15 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[4:7], 0 offen glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[4:7], 0 offen glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB12_4
; GFX10-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v8, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v6
+; GFX10-NEXT: v_mov_b32_e32 v6, v4
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -3790,19 +3742,19 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX10-NEXT: s_cbranch_execnz .LBB12_3
; GFX10-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX90A-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX90A-NEXT: v_and_b32_e32 v9, -4, v4
; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v8, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX90A-NEXT: v_not_b32_e32 v11, v6
+; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v8, s4
+; GFX90A-NEXT: v_not_b32_e32 v10, v4
; GFX90A-NEXT: s_mov_b64 s[6:7], exec
; GFX90A-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
@@ -3814,24 +3766,24 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_nop 0
-; GFX90A-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX90A-NEXT: buffer_load_dword v7, v9, s[8:11], 0 offen
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2:
; GFX90A-NEXT: s_mov_b64 exec, s[6:7]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_max_f16_e32 v11, v5, v5
; GFX90A-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB12_4 Depth 2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX90A-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX90A-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX90A-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX90A-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v8, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX90A-NEXT: v_min_f16_e32 v4, v4, v11
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v8, v4
+; GFX90A-NEXT: v_and_or_b32 v6, v7, v10, v4
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
; GFX90A-NEXT: v_readfirstlane_b32 s8, v0
@@ -3843,33 +3795,33 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX90A-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v9, s[8:11], 0 offen glc
; GFX90A-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_4
; GFX90A-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX90A-NEXT: s_mov_b64 exec, s[12:13]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v7, v8
+; GFX90A-NEXT: v_mov_b32_e32 v7, v4
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_3
; GFX90A-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v8, v4
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_add_u32_e32 v4, 0x200, v4
-; GFX908-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX908-NEXT: v_and_b32_e32 v8, -4, v4
; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v7, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX908-NEXT: v_not_b32_e32 v11, v6
+; GFX908-NEXT: v_lshlrev_b32_e64 v4, v7, s4
+; GFX908-NEXT: v_not_b32_e32 v9, v4
; GFX908-NEXT: s_mov_b64 s[6:7], exec
; GFX908-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -3881,25 +3833,25 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_nop 0
-; GFX908-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX908-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
; GFX908-NEXT: ; %bb.2:
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_max_f16_e32 v10, v5, v5
; GFX908-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB12_4 Depth 2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX908-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX908-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX908-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX908-NEXT: v_and_or_b32 v6, v7, v11, v6
+; GFX908-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX908-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX908-NEXT: v_min_f16_e32 v4, v4, v10
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX908-NEXT: v_and_or_b32 v5, v6, v9, v4
; GFX908-NEXT: s_mov_b64 s[12:13], exec
-; GFX908-NEXT: v_mov_b32_e32 v9, v7
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v4, v5
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -3911,33 +3863,33 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_4
; GFX908-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
+; GFX908-NEXT: v_mov_b32_e32 v6, v4
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB12_3
; GFX908-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x200, v4
-; GFX8-NEXT: v_and_b32_e32 v10, -4, v4
+; GFX8-NEXT: v_and_b32_e32 v8, -4, v4
; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v7, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v6, v4, s4
-; GFX8-NEXT: v_not_b32_e32 v11, v6
+; GFX8-NEXT: v_lshlrev_b32_e64 v4, v7, s4
+; GFX8-NEXT: v_not_b32_e32 v9, v4
; GFX8-NEXT: s_mov_b64 s[6:7], exec
; GFX8-NEXT: .LBB12_1: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -3949,26 +3901,26 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_nop 0
-; GFX8-NEXT: buffer_load_dword v7, v10, s[8:11], 0 offen
+; GFX8-NEXT: buffer_load_dword v6, v8, s[8:11], 0 offen
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_1
; GFX8-NEXT: ; %bb.2:
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_max_f16_e32 v10, v5, v5
; GFX8-NEXT: .LBB12_3: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB12_4 Depth 2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, v4, v7
-; GFX8-NEXT: v_max_f16_e32 v6, v6, v6
-; GFX8-NEXT: v_max_f16_e32 v8, v5, v5
-; GFX8-NEXT: v_min_f16_e32 v6, v6, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, v4, v6
-; GFX8-NEXT: v_and_b32_e32 v8, v7, v11
-; GFX8-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, v7, v6
+; GFX8-NEXT: v_max_f16_e32 v4, v4, v4
+; GFX8-NEXT: v_min_f16_e32 v4, v4, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX8-NEXT: v_and_b32_e32 v5, v6, v9
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v4
; GFX8-NEXT: s_mov_b64 s[12:13], exec
-; GFX8-NEXT: v_mov_b32_e32 v9, v7
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: .LBB12_4: ; Parent Loop BB12_3 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -3980,21 +3932,21 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[8:9], v10, s[8:11], 0 offen glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v8, s[8:11], 0 offen glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB12_4
; GFX8-NEXT: ; %bb.5: ; in Loop: Header=BB12_3 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v8, v7
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
+; GFX8-NEXT: v_mov_b32_e32 v6, v4
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB12_3
; GFX8-NEXT: ; %bb.6: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, v4, v8
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, v7, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -4151,27 +4103,27 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v4
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -4181,56 +4133,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB13_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v4, s4
+; GFX942-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v5, s6
-; GFX942-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v5
; GFX942-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX942-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4240,32 +4193,33 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
@@ -4275,97 +4229,95 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB13_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v5
; GFX10-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX10-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX10-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX10-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX10-NEXT: v_mov_b32_e32 v3, v1
; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, v2
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB13_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, s5, v2
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, s4, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v4, s4
+; GFX90A-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v5
; GFX90A-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX90A-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v5, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4375,39 +4327,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v5
; GFX908-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX908-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v2, v2, v0, s9
+; GFX908-NEXT: v_add3_u32 v2, v2, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX908-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4417,41 +4368,40 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX908-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v5
; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v2, s8, v1
+; GFX8-NEXT: v_and_b32_e32 v2, s7, v1
; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4461,39 +4411,38 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX7-NEXT: v_min_f32_e32 v0, v0, v5
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX7-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: v_mov_b32_e32 v3, v1
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4503,40 +4452,39 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX7-NEXT: s_cbranch_execnz .LBB13_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v4, s4
+; GFX6-NEXT: buffer_load_dword v1, v4, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
; GFX6-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX6-NEXT: v_min_f32_e32 v0, v0, v5
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
+; GFX6-NEXT: v_and_b32_e32 v2, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
; GFX6-NEXT: v_mov_b32_e32 v3, v1
; GFX6-NEXT: v_mov_b32_e32 v2, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[2:3], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
@@ -4546,7 +4494,7 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX6-NEXT: s_cbranch_execnz .LBB13_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(7) %ptr, i32 256
@@ -4563,85 +4511,86 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_and_b32 s5, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v1, s4
-; GFX12-NEXT: s_lshl_b32 s5, s5, 3
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s6, 0xffff, s5
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_not_b32 s7, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null offen
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX12-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_cbranch_execnz .LBB14_1
; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: s_addk_i32 s16, 0x200
-; GFX942-NEXT: s_and_b32 s6, s16, -4
-; GFX942-NEXT: v_mov_b32_e32 v1, s6
-; GFX942-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen
+; GFX942-NEXT: s_and_b32 s4, s16, -4
+; GFX942-NEXT: v_mov_b32_e32 v2, s4
+; GFX942-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen
; GFX942-NEXT: s_and_b32 s4, s16, 3
-; GFX942-NEXT: s_lshl_b32 s7, s4, 3
-; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX942-NEXT: s_not_b32 s8, s4
+; GFX942-NEXT: s_lshl_b32 s6, s4, 3
+; GFX942-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX942-NEXT: s_not_b32 s7, s4
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX942-NEXT: s_movk_i32 s9, 0x7fff
+; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: v_mov_b32_e32 v3, s6
-; GFX942-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: buffer_wbl2 sc1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v3
; GFX942-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX942-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX942-NEXT: v_and_or_b32 v0, v1, s8, v0
+; GFX942-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX942-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen sc0
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[0:3], 0 offen sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -4657,123 +4606,122 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_and_b32 s5, s16, 3
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: s_lshl_b32 s5, s5, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX11-NEXT: s_not_b32 s7, s6
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 offen
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s5, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s5, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen glc
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v3
-; GFX11-NEXT: s_or_b32 s6, vcc_lo, s6
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_cbranch_execnz .LBB14_1
; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: s_addk_i32 s20, 0x200
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX10-NEXT: s_and_b32 s4, s20, -4
-; GFX10-NEXT: s_and_b32 s5, s20, 3
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: s_lshl_b32 s5, s5, 3
-; GFX10-NEXT: s_lshl_b32 s6, 0xffff, s5
-; GFX10-NEXT: s_not_b32 s7, s6
-; GFX10-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
+; GFX10-NEXT: s_and_b32 s4, s20, 3
+; GFX10-NEXT: s_lshl_b32 s4, s4, 3
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX10-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
+; GFX10-NEXT: s_not_b32 s6, s5
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v5, s4
+; GFX10-NEXT: v_lshrrev_b32_sdwa v0, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX10-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc_lo
-; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX10-NEXT: v_and_or_b32 v0, v1, s7, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, v1
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX10-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX10-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, v3
-; GFX10-NEXT: s_or_b32 s6, vcc_lo, s6
-; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
+; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB14_1
; GFX10-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_addk_i32 s20, 0x200
-; GFX90A-NEXT: s_and_b32 s6, s20, -4
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
-; GFX90A-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX90A-NEXT: s_and_b32 s4, s20, -4
+; GFX90A-NEXT: v_mov_b32_e32 v2, s4
+; GFX90A-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX90A-NEXT: s_and_b32 s4, s20, 3
-; GFX90A-NEXT: s_lshl_b32 s7, s4, 3
-; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX90A-NEXT: s_not_b32 s8, s4
+; GFX90A-NEXT: s_lshl_b32 s6, s4, 3
+; GFX90A-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX90A-NEXT: s_not_b32 s7, s4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX90A-NEXT: s_movk_i32 s9, 0x7fff
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
; GFX90A-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s9
+; GFX90A-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
-; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX90A-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX90A-NEXT: v_mov_b32_e32 v3, s6
+; GFX90A-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX90A-NEXT: v_and_or_b32 v0, v1, s7, v0
; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen glc
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
@@ -4789,37 +4737,36 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: s_addk_i32 s20, 0x200
-; GFX908-NEXT: s_and_b32 s6, s20, -4
-; GFX908-NEXT: v_mov_b32_e32 v1, s6
-; GFX908-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX908-NEXT: s_and_b32 s4, s20, -4
+; GFX908-NEXT: v_mov_b32_e32 v2, s4
+; GFX908-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX908-NEXT: s_and_b32 s4, s20, 3
-; GFX908-NEXT: s_lshl_b32 s7, s4, 3
-; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX908-NEXT: s_not_b32 s8, s4
+; GFX908-NEXT: s_lshl_b32 s6, s4, 3
+; GFX908-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX908-NEXT: s_not_b32 s7, s4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
-; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX908-NEXT: s_movk_i32 s9, 0x7fff
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
-; GFX908-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v0, s9
+; GFX908-NEXT: v_lshrrev_b32_sdwa v0, s6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX908-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX908-NEXT: v_add3_u32 v4, v4, v0, s8
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cndmask_b32_e32 v0, v3, v4, vcc
-; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX908-NEXT: v_and_or_b32 v0, v1, s8, v0
-; GFX908-NEXT: v_mov_b32_e32 v5, s6
-; GFX908-NEXT: v_mov_b32_e32 v4, v1
-; GFX908-NEXT: v_mov_b32_e32 v3, v0
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX908-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc
+; GFX908-NEXT: v_lshlrev_b32_sdwa v0, s6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX908-NEXT: v_and_or_b32 v0, v1, s7, v0
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v1, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4830,39 +4777,38 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_addk_i32 s20, 0x200
-; GFX8-NEXT: s_and_b32 s6, s20, -4
-; GFX8-NEXT: v_mov_b32_e32 v1, s6
-; GFX8-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX8-NEXT: s_and_b32 s4, s20, -4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX8-NEXT: s_and_b32 s4, s20, 3
-; GFX8-NEXT: s_lshl_b32 s7, s4, 3
-; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s7
-; GFX8-NEXT: s_not_b32 s8, s4
+; GFX8-NEXT: s_lshl_b32 s6, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_not_b32 s7, s4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_sdwa v4, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f32_e32 v4, v4, v2
-; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_sdwa v5, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc
-; GFX8-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, v0
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v7, vcc
+; GFX8-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, v1
+; GFX8-NEXT: v_mov_b32_e32 v4, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4873,37 +4819,36 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_addk_i32 s20, 0x200
-; GFX7-NEXT: s_and_b32 s6, s20, -4
-; GFX7-NEXT: v_mov_b32_e32 v1, s6
-; GFX7-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX7-NEXT: s_and_b32 s4, s20, -4
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX7-NEXT: s_and_b32 s4, s20, 3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: s_lshl_b32 s7, s4, 3
-; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX7-NEXT: s_lshl_b32 s6, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: s_not_b32 s8, s4
+; GFX7-NEXT: s_not_b32 s7, s4
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX7-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX7-NEXT: v_min_f32_e32 v0, v0, v3
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, v0
-; GFX7-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX7-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
+; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v1, v3
+; GFX7-NEXT: v_mov_b32_e32 v1, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4914,38 +4859,37 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_addk_i32 s20, 0x200
-; GFX6-NEXT: s_and_b32 s6, s20, -4
-; GFX6-NEXT: v_mov_b32_e32 v1, s6
-; GFX6-NEXT: buffer_load_dword v1, v1, s[16:19], 0 offen
+; GFX6-NEXT: s_and_b32 s4, s20, -4
+; GFX6-NEXT: v_mov_b32_e32 v2, s4
+; GFX6-NEXT: buffer_load_dword v1, v2, s[16:19], 0 offen
; GFX6-NEXT: s_and_b32 s4, s20, 3
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: s_lshl_b32 s7, s4, 3
-; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s6, s4, 3
+; GFX6-NEXT: s_lshl_b32 s4, 0xffff, s6
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: s_not_b32 s8, s4
+; GFX6-NEXT: s_not_b32 s7, s4
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX6-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s7, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v0
-; GFX6-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX6-NEXT: v_min_f32_e32 v0, v0, v3
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, s8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s7, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_mov_b32_e32 v5, s6
-; GFX6-NEXT: v_mov_b32_e32 v4, v1
-; GFX6-NEXT: v_mov_b32_e32 v3, v0
-; GFX6-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen glc
+; GFX6-NEXT: v_and_b32_e32 v4, s7, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s6, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
+; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_mov_b32_e32 v1, v3
+; GFX6-NEXT: v_mov_b32_e32 v1, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB14_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5660,28 +5604,28 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v1, v1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v2, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v1, v4, v4
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v3, v1, v0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v4, v0, v2
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5697,31 +5641,31 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v1, v4, v4
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v3, v1, v0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v4, v0, v2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5733,25 +5677,25 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_pk_max_f16 v1, v2, v2
; GFX942-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_pk_min_f16 v4, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -5767,29 +5711,29 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v1, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v4, v4
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_min_f16 v3, v1, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5801,32 +5745,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v1, v4, v4
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_min_f16 v3, v1, v0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -5838,32 +5782,31 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_max_f16 v1, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v1, v0
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX10-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v4
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB16_1
@@ -5874,23 +5817,22 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_pk_max_f16 v1, v2, v2
; GFX90A-NEXT: v_pk_max_f16 v0, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_pk_min_f16 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -5906,29 +5848,28 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v0, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v5, v5
+; GFX908-NEXT: v_pk_min_f16 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB16_1
@@ -5939,33 +5880,32 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v6, v0
-; GFX8-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v1, v1, v3
+; GFX8-NEXT: v_or_b32_e32 v5, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB16_1
@@ -5984,27 +5924,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6027,27 +5967,27 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__amdgpu_no
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6072,28 +6012,28 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v1, v0, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v3, v2, v2
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v1, v3, v1
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6113,27 +6053,27 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v1, v0, v0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v3, v2, v2
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v1, v3, v1
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6149,26 +6089,27 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX942-NEXT: v_mov_b32_e32 v1, s16
; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v0
; GFX942-NEXT: s_mov_b64 s[4:5], 0
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_pk_min_f16 v2, v2, v1
+; GFX942-NEXT: v_pk_max_f16 v0, v1, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX942-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6178,29 +6119,29 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s16
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v0, v1, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_pk_min_f16 v1, v3, v1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6216,28 +6157,28 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v3, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v2, v0, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, s16
; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v0, v1, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_pk_min_f16 v1, v3, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -6255,25 +6196,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_pk_max_f16 v2, v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, s20
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
+; GFX10-NEXT: v_pk_max_f16 v0, v1, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_pk_min_f16 v1, v3, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB17_1
@@ -6288,24 +6229,24 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_pk_max_f16 v2, v0, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v2, v1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6319,25 +6260,25 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX908-NEXT: v_mov_b32_e32 v1, s20
; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_pk_max_f16 v2, v0, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v1, v0, v0
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX908-NEXT: v_pk_min_f16 v1, v3, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX908-NEXT: v_pk_min_f16 v0, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_mov_b32_e32 v4, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v1
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6349,31 +6290,31 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_max_f16_sdwa v2, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v3, v0, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v4, v0, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_min_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v3, v5, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_max_f16_sdwa v0, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6391,27 +6332,27 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s20
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX7-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX7-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_mov_b32_e32 v7, s20
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v2, v5
-; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX7-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX7-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, v6
+; GFX7-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
@@ -6431,27 +6372,27 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2f16__offset__amdgpu_no_fin
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: s_add_i32 s6, s20, 0x400
; GFX6-NEXT: s_mov_b64 s[4:5], 0
+; GFX6-NEXT: v_mov_b32_e32 v4, s6
; GFX6-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_min_f32_e32 v5, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v5
-; GFX6-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_or_b32_e32 v5, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_mov_b32_e32 v7, s6
-; GFX6-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX6-NEXT: v_mov_b32_e32 v1, v4
-; GFX6-NEXT: v_mov_b32_e32 v2, v5
-; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v7, s[16:19], 0 offen glc
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v7, v6
+; GFX6-NEXT: v_or_b32_e32 v6, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v5, v7, v1
+; GFX6-NEXT: v_mov_b32_e32 v1, v5
+; GFX6-NEXT: v_mov_b32_e32 v2, v6
+; GFX6-NEXT: buffer_atomic_cmpswap v[1:2], v4, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v5
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v1, v6
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
@@ -6486,7 +6427,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX12-TRUE16-NEXT: ; %bb.2:
@@ -6511,21 +6452,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: ; %bb.4:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v6, v5, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v7, v8, v8
+; GFX12-TRUE16-NEXT: v_pk_max_num_f16 v5, v7, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v7, v7, v6
+; GFX12-TRUE16-NEXT: v_pk_min_num_f16 v6, v5, v8
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6537,14 +6478,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6552,7 +6493,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6600,21 +6541,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: ; %bb.4:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v8, v5, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v6, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v7, v8, v8
+; GFX12-FAKE16-NEXT: v_pk_max_num_f16 v5, v7, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v7, v7, v6
+; GFX12-FAKE16-NEXT: v_pk_min_num_f16 v6, v5, v8
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6626,14 +6567,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6641,7 +6582,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6683,13 +6624,13 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_pk_max_f16 v5, v5, v5
; GFX942-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX942-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX942-NEXT: v_pk_max_f16 v7, v9, v9
+; GFX942-NEXT: v_pk_max_f16 v6, v9, v9
; GFX942-NEXT: s_mov_b64 s[8:9], exec
-; GFX942-NEXT: v_pk_min_f16 v8, v7, v6
+; GFX942-NEXT: v_pk_min_f16 v8, v6, v5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
@@ -6736,7 +6677,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_1
; GFX11-TRUE16-NEXT: ; %bb.2:
@@ -6758,20 +6699,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: ; %bb.4:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v6, v5, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX11-TRUE16-NEXT: v_pk_max_f16 v5, v7, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_pk_min_f16 v7, v7, v6
+; GFX11-TRUE16-NEXT: v_pk_min_f16 v6, v5, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-TRUE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6782,14 +6723,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6798,7 +6739,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6837,20 +6778,20 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: ; %bb.4:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v8, v5, v5
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v6, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX11-FAKE16-NEXT: v_pk_max_f16 v5, v7, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_pk_min_f16 v7, v7, v6
+; GFX11-FAKE16-NEXT: v_pk_min_f16 v6, v5, v8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-FAKE16-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -6861,14 +6802,14 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -6877,7 +6818,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB18_5
; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6917,18 +6858,18 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX10-NEXT: v_pk_max_f16 v8, v5, v5
; GFX10-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX10-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX10-NEXT: v_pk_max_f16 v7, v8, v8
+; GFX10-NEXT: v_pk_max_f16 v5, v7, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_pk_min_f16 v7, v7, v6
+; GFX10-NEXT: v_pk_min_f16 v6, v5, v8
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -6939,15 +6880,15 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB18_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -6956,7 +6897,7 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX10-NEXT: s_cbranch_execnz .LBB18_5
; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -6998,12 +6939,12 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_pk_max_f16 v5, v5, v5
; GFX90A-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX90A-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v7, v9, v9
-; GFX90A-NEXT: v_pk_min_f16 v8, v7, v6
+; GFX90A-NEXT: v_pk_max_f16 v6, v9, v9
+; GFX90A-NEXT: v_pk_min_f16 v8, v6, v5
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
; GFX90A-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
@@ -7071,17 +7012,17 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b32 s4, 0x5040100
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_pk_max_f16 v8, v5, v5
; GFX908-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX908-NEXT: v_pk_max_f16 v6, v5, v5
-; GFX908-NEXT: v_pk_max_f16 v7, v8, v8
-; GFX908-NEXT: v_pk_min_f16 v7, v7, v6
+; GFX908-NEXT: v_pk_max_f16 v5, v7, v7
+; GFX908-NEXT: v_pk_min_f16 v6, v5, v8
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -7093,21 +7034,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB18_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB18_5
; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7147,21 +7088,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_max_f16_sdwa v8, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v9, v5, v5
; GFX8-NEXT: .LBB18_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB18_6 Depth 2
-; GFX8-NEXT: v_max_f16_sdwa v6, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_sdwa v7, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v7, v5, v5
-; GFX8-NEXT: v_max_f16_e32 v9, v8, v8
-; GFX8-NEXT: v_min_f16_e32 v7, v9, v7
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v6
+; GFX8-NEXT: v_max_f16_sdwa v5, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v7, v7
+; GFX8-NEXT: v_min_f16_sdwa v5, v5, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v9
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v5
; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
; GFX8-NEXT: .LBB18_6: ; Parent Loop BB18_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -7173,21 +7114,21 @@ define <2 x half> @buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB18_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB18_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB18_5
; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2f16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -7380,49 +7321,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v1, v0
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v5, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v1, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7438,50 +7381,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v1, v5, v1
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v1, v1, v3
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -7493,46 +7436,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
-; GFX942-NEXT: v_mov_b32_e32 v0, s16
-; GFX942-NEXT: buffer_load_ushort v3, v1, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v0, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v1, v4, v1
-; GFX942-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX942-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
+; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX942-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX942-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX942-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[6:7]
+; GFX942-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB19_1
@@ -7543,51 +7485,52 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v0, s16
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v3, v0, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v1, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v0, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_lshlrev_b32 v3, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v5, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -7600,51 +7543,51 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v1, s16
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s16
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_load_u16 v0, v0, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_and_b32 v1, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v1, v5, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v0, v3, v0 :: v_dual_cndmask_b32 v1, v5, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v7, v9 :: v_dual_lshlrev_b32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v6 :: v_dual_mov_b32 v0, v5
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -7657,46 +7600,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, v0
-; GFX10-NEXT: v_mov_b32_e32 v0, s20
; GFX10-NEXT: v_mov_b32_e32 v1, s20
+; GFX10-NEXT: v_mov_b32_e32 v2, s20
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_ushort v3, v0, s[16:19], 0 offen offset:1026
-; GFX10-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
+; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v0, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_mov_b32_e32 v4, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX10-NEXT: v_min_f32_e32 v1, v5, v1
-; GFX10-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX10-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX10-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX10-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v6, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v1, v4
-; GFX10-NEXT: v_mov_b32_e32 v0, v3
-; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, v8, s4
+; GFX10-NEXT: v_perm_b32 v5, v1, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v1, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
+; GFX10-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB19_1
@@ -7707,44 +7649,43 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v0, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v1, v4, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v4, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v1, v0, s9
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_mov_b32_e32 v7, v0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v7
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v7
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX90A-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX90A-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v5, v6, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v6, v1, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[6:7], v[6:7] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB19_1
@@ -7755,45 +7696,44 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX908-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
-; GFX908-NEXT: v_mov_b32_e32 v0, s20
-; GFX908-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v0, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX908-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v3, v3, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v1, v0, s9
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX908-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX908-NEXT: v_add3_u32 v5, v5, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v1, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v5, v1, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
+; GFX908-NEXT: v_mov_b32_e32 v1, v6
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB19_1
@@ -7804,46 +7744,45 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s20
; GFX8-NEXT: v_mov_b32_e32 v1, s20
-; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1026
-; GFX8-NEXT: buffer_load_ushort v0, v0, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_mov_b32_e32 v2, s20
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX8-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_min_f32_e32 v1, v1, v3
+; GFX8-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v1
+; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v5, v7, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_alignbit_b32 v3, v1, v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_alignbit_b32 v5, v1, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
+; GFX8-NEXT: v_mov_b32_e32 v1, v6
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB19_1
@@ -7854,50 +7793,50 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_min_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v0, 1.0, v4
; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v3
; GFX7-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -7906,49 +7845,49 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__amdgpu
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_min_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB19_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX6-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v3
+; GFX6-NEXT: v_mul_f32_e32 v0, 1.0, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v2
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v3
; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
; GFX6-NEXT: s_waitcnt expcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -7965,49 +7904,47 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1026
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], null offen offset:1026
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v1, v3, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v5, v5, v3 :: v_dual_min_num_f32 v0, v0, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8024,49 +7961,47 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX12-FAKE16-NEXT: s_clause 0x1
; GFX12-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], null offen offset:1024
-; GFX12-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], null offen offset:1026
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x1
; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX12-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v1, v3, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
+; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v5, v3 :: v_dual_min_num_f32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -8080,44 +8015,44 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v2, s16
; GFX942-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-NEXT: buffer_load_ushort v3, v2, s[0:3], 0 offen offset:1026
-; GFX942-NEXT: buffer_load_ushort v4, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: buffer_load_ushort v4, v2, s[0:3], 0 offen offset:1026
+; GFX942-NEXT: buffer_load_ushort v5, v1, s[0:3], 0 offen offset:1024
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX942-NEXT: s_mov_b64 s[6:7], 0
; GFX942-NEXT: s_movk_i32 s8, 0x7fff
; GFX942-NEXT: s_mov_b32 s9, 0x7060302
; GFX942-NEXT: s_waitcnt vmcnt(1)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX942-NEXT: v_mov_b32_e32 v4, s16
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX942-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX942-NEXT: v_min_f32_e32 v2, v5, v4
-; GFX942-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX942-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX942-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX942-NEXT: v_mov_b32_e32 v6, s16
-; GFX942-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX942-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX942-NEXT: v_mov_b64_e32 v[4:5], v[2:3]
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX942-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[0:3], 0 offen offset:1024 sc0
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX942-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
+; GFX942-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[0:3], 0 offen offset:1024 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX942-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v3, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v6
; GFX942-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-NEXT: s_cbranch_execnz .LBB20_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8127,51 +8062,48 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_and_b32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1026
-; GFX11-TRUE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v4, v1, s[0:3], 0 offen offset:1026
+; GFX11-TRUE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: v_dual_min_f32 v5, v5, v3 :: v_dual_min_f32 v0, v0, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -8185,50 +8117,48 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s16 :: v_dual_mov_b32 v2, s16
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: buffer_load_u16 v1, v1, s[0:3], 0 offen offset:1024
-; GFX11-FAKE16-NEXT: buffer_load_u16 v2, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: buffer_load_u16 v4, v2, s[0:3], 0 offen offset:1026
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s16
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, s16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v2 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v5, v3 :: v_dual_min_f32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v1 :: v_dual_mov_b32 v5, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
@@ -8247,39 +8177,39 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1026
; GFX10-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1024
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX10-NEXT: s_waitcnt vmcnt(1)
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_mov_b32_e32 v4, s20
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v4
-; GFX10-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX10-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v1, v1
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v4, v6, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v5, s20
-; GFX10-NEXT: v_perm_b32 v1, v3, v1, 0x7060302
-; GFX10-NEXT: v_mov_b32_e32 v4, v2
-; GFX10-NEXT: v_mov_b32_e32 v3, v1
-; GFX10-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v6, v1
+; GFX10-NEXT: v_mov_b32_e32 v5, v0
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, v5
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB20_1
@@ -8292,42 +8222,42 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v2, s20
; GFX90A-NEXT: v_mov_b32_e32 v1, s20
-; GFX90A-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX90A-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX90A-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX90A-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: s_waitcnt vmcnt(1)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_or_b32_sdwa v3, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX90A-NEXT: v_mov_b32_e32 v4, s20
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX90A-NEXT: v_min_f32_e32 v2, v5, v4
-; GFX90A-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX90A-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX90A-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v2, v1, s9
-; GFX90A-NEXT: v_mov_b32_e32 v6, s20
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
-; GFX90A-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[0:1], v[0:1] op_sel:[0,1]
+; GFX90A-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[16:19], 0 offen offset:1024 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v6, v1
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v4
+; GFX90A-NEXT: v_mov_b32_e32 v1, v6
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8339,43 +8269,43 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v2, s20
; GFX908-NEXT: v_mov_b32_e32 v1, s20
-; GFX908-NEXT: buffer_load_ushort v3, v2, s[16:19], 0 offen offset:1026
-; GFX908-NEXT: buffer_load_ushort v4, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
+; GFX908-NEXT: buffer_load_ushort v5, v1, s[16:19], 0 offen offset:1024
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: s_waitcnt vmcnt(1)
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_or_b32_sdwa v2, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_or_b32_sdwa v1, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX908-NEXT: v_mov_b32_e32 v4, s20
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX908-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v4
-; GFX908-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v4, v4, v1, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX908-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v1, v3, v1, s9
-; GFX908-NEXT: v_mov_b32_e32 v6, s20
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_mov_b32_e32 v3, v1
-; GFX908-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_mov_b32_e32 v6, v1
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB20_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8387,44 +8317,44 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s20
; GFX8-NEXT: v_mov_b32_e32 v2, s20
-; GFX8-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
+; GFX8-NEXT: buffer_load_ushort v4, v2, s[16:19], 0 offen offset:1026
; GFX8-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v4
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v4, s20
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v4
-; GFX8-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fff, v4
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v4, v5, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX8-NEXT: v_mov_b32_e32 v6, s20
-; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: buffer_atomic_cmpswap v[3:4], v6, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v3
+; GFX8-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v0
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
+; GFX8-NEXT: v_mov_b32_e32 v6, v1
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB20_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8434,43 +8364,43 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fine_grained_memory:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: v_mov_b32_e32 v2, s20
+; GFX7-NEXT: v_mov_b32_e32 v1, s20
; GFX7-NEXT: buffer_load_ushort v2, v2, s[16:19], 0 offen offset:1026
-; GFX7-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX7-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX7-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_mov_b32_e32 v2, s20
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX7-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_min_f32_e32 v4, v4, v0
-; GFX7-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX7-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX7-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX7-NEXT: v_mov_b32_e32 v6, s20
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX7-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-NEXT: v_min_f32_e32 v5, v5, v0
+; GFX7-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX7-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX7-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX7-NEXT: v_mov_b32_e32 v6, v4
; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
-; GFX7-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen offset:1024 glc
+; GFX7-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen offset:1024 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB20_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8482,42 +8412,42 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_v2bf16__offset__amdgpu_no_fi
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s20
; GFX6-NEXT: buffer_load_ushort v2, v1, s[16:19], 0 offen offset:1026
-; GFX6-NEXT: buffer_load_ushort v1, v1, s[16:19], 0 offen offset:1024
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX6-NEXT: buffer_load_ushort v3, v1, s[16:19], 0 offen offset:1024
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
+; GFX6-NEXT: s_add_i32 s6, s20, 0x400
+; GFX6-NEXT: v_mul_f32_e32 v1, 1.0, v1
; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v0
; GFX6-NEXT: s_mov_b64 s[4:5], 0
-; GFX6-NEXT: s_add_i32 s6, s20, 0x400
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_mov_b32_e32 v2, s6
; GFX6-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX6-NEXT: v_mul_f32_e32 v4, 1.0, v4
; GFX6-NEXT: v_mul_f32_e32 v3, 1.0, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_min_f32_e32 v4, v4, v0
-; GFX6-NEXT: v_min_f32_e32 v5, v5, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v2, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v5, 16
-; GFX6-NEXT: v_mov_b32_e32 v6, s6
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_f32_e32 v5, v5, v0
+; GFX6-NEXT: v_min_f32_e32 v6, v6, v1
+; GFX6-NEXT: v_alignbit_b32 v4, v4, v3, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_alignbit_b32 v3, v3, v6, 16
+; GFX6-NEXT: v_mov_b32_e32 v6, v4
; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[4:5], v6, s[16:19], 0 offen glc
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v2, s[16:19], 0 offen glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v5
; GFX6-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
; GFX6-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX6-NEXT: s_cbranch_execnz .LBB20_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8550,7 +8480,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], null offen offset:1024
+; GFX12-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], null offen offset:1024
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s5, s5
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX12-TRUE16-NEXT: ; %bb.2:
@@ -8575,42 +8505,41 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: ; %bb.4:
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
; GFX12-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX12-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v7, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v7, v10, v9
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v6, v6, v9 :: v_dual_min_num_f32 v5, v5, v8
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX12-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8622,14 +8551,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8637,7 +8566,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX12-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8685,39 +8614,38 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: ; %bb.4:
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s4
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX12-FAKE16-NEXT: s_mov_b32 s4, 0
; GFX12-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX12-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX12-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v7, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v7, v10, v9
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX12-FAKE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v6, v6, v9 :: v_dual_min_num_f32 v5, v5, v8
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX12-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX12-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8729,14 +8657,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX12-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], null offen offset:1024 th:TH_ATOMIC_RETURN
; GFX12-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX12-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX12-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8744,7 +8672,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX12-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8786,14 +8714,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_perm_b32 v9, v7, v6, s0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX942-NEXT: s_movk_i32 s10, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX942-NEXT: s_mov_b32 s11, 0x7060302
; GFX942-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Loop Header: Depth=1
; GFX942-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX942-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX942-NEXT: v_min_f32_e32 v6, v7, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v10
; GFX942-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX942-NEXT: v_add3_u32 v7, v7, v6, s10
; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v6
@@ -8802,15 +8731,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX942-NEXT: v_min_f32_e32 v7, v8, v7
+; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX942-NEXT: v_min_f32_e32 v7, v7, v5
; GFX942-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX942-NEXT: v_add3_u32 v8, v8, v7, s10
-; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX942-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX942-NEXT: v_perm_b32 v8, v7, v6, s11
; GFX942-NEXT: v_mov_b64_e32 v[6:7], v[8:9]
; GFX942-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
@@ -8856,7 +8784,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
-; GFX11-TRUE16-NEXT: buffer_load_u16 v8, v4, s[0:3], 0 offen offset:1024
+; GFX11-TRUE16-NEXT: buffer_load_u16 v7, v4, s[0:3], 0 offen offset:1024
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_1
; GFX11-TRUE16-NEXT: ; %bb.2:
@@ -8878,39 +8806,39 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: ; %bb.4:
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-TRUE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v7
; GFX11-TRUE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v7, v6
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v7, v10, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-TRUE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-TRUE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -8921,14 +8849,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-TRUE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-TRUE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-TRUE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
; GFX11-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -8936,8 +8864,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -8976,36 +8905,36 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: ; %bb.4:
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
+; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Loop Header: Depth=1
; GFX11-FAKE16-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX11-FAKE16-NEXT: s_mov_b32 s5, exec_lo
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v7, v6
; GFX11-FAKE16-NEXT: s_mov_b32 s6, s5
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v7, v10, v9
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-FAKE16-NEXT: v_dual_min_f32 v6, v6, v9 :: v_dual_min_f32 v5, v5, v8
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, v7 :: v_dual_mov_b32 v7, v8
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v7
; GFX11-FAKE16-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX11-FAKE16-NEXT: ; => This Inner Loop Header: Depth=2
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
@@ -9016,14 +8945,14 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[0:1]
; GFX11-FAKE16-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[2:3]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[6:7], v4, s[0:3], 0 offen offset:1024 glc
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[5:6], v4, s[0:3], 0 offen offset:1024 glc
; GFX11-FAKE16-NEXT: s_and_not1_wrexec_b32 s6, s6
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_6
; GFX11-FAKE16-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s5
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v8, v6
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v7, v5
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
; GFX11-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -9031,8 +8960,9 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB21_5
; GFX11-FAKE16-NEXT: ; %bb.8: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v6
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v5
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9072,32 +9002,32 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
+; GFX10-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX10-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Loop Header: Depth=1
; GFX10-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX10-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
; GFX10-NEXT: s_mov_b32 s9, exec_lo
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: v_min_f32_e32 v6, v7, v6
; GFX10-NEXT: s_mov_b32 s10, s9
-; GFX10-NEXT: v_min_f32_e32 v7, v10, v9
-; GFX10-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX10-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v8
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v9
+; GFX10-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX10-NEXT: v_bfe_u32 v11, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_or_b32_e32 v13, 0x400000, v6
+; GFX10-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v11, v11, v6, 0x7fff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX10-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX10-NEXT: v_add3_u32 v9, v9, v6, 0x7fff
-; GFX10-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v10, v12, vcc_lo
-; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v11, v13, vcc_lo
+; GFX10-NEXT: v_perm_b32 v6, v6, v5, 0x7060302
+; GFX10-NEXT: v_mov_b32_e32 v5, v6
; GFX10-NEXT: v_mov_b32_e32 v6, v7
-; GFX10-NEXT: v_mov_b32_e32 v7, v8
; GFX10-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX10-NEXT: ; => This Inner Loop Header: Depth=2
; GFX10-NEXT: v_readfirstlane_b32 s4, v0
@@ -9108,15 +9038,15 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[4:5], v[0:1]
; GFX10-NEXT: v_cmpx_eq_u64_e32 s[6:7], v[2:3]
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[4:7], 0 offen offset:1024 glc
+; GFX10-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[4:7], 0 offen offset:1024 glc
; GFX10-NEXT: s_andn2_wrexec_b32 s10, s10
; GFX10-NEXT: s_cbranch_execnz .LBB21_6
; GFX10-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_mov_b32 exec_lo, s9
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v8
-; GFX10-NEXT: v_mov_b32_e32 v8, v6
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v7
+; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: s_or_b32 s8, vcc_lo, s8
@@ -9125,7 +9055,7 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX10-NEXT: s_cbranch_execnz .LBB21_5
; GFX10-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, v6
+; GFX10-NEXT: v_mov_b32_e32 v0, v5
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9167,27 +9097,27 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_perm_b32 v9, v7, v6, s4
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX90A-NEXT: s_movk_i32 s14, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX90A-NEXT: s_mov_b32 s15, 0x7060302
; GFX90A-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Loop Header: Depth=1
; GFX90A-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX90A-NEXT: v_lshlrev_b32_e32 v7, 16, v9
-; GFX90A-NEXT: v_min_f32_e32 v6, v7, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v10
; GFX90A-NEXT: v_bfe_u32 v7, v6, 16, 1
; GFX90A-NEXT: v_add3_u32 v7, v7, v6, s14
; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v6
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_and_b32_e32 v8, 0xffff0000, v9
-; GFX90A-NEXT: v_min_f32_e32 v7, v8, v7
+; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v9
+; GFX90A-NEXT: v_min_f32_e32 v7, v7, v5
; GFX90A-NEXT: v_bfe_u32 v8, v7, 16, 1
; GFX90A-NEXT: v_add3_u32 v8, v8, v7, s14
-; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX90A-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc
+; GFX90A-NEXT: v_cndmask_b32_e32 v7, v8, v11, vcc
; GFX90A-NEXT: v_perm_b32 v8, v7, v6, s15
; GFX90A-NEXT: s_mov_b64 s[12:13], exec
; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[8:9], v[8:9] op_sel:[0,1]
@@ -9256,33 +9186,33 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_mov_b64 exec, s[6:7]
; GFX908-NEXT: s_mov_b32 s4, 0x5040100
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_perm_b32 v8, v7, v6, s4
+; GFX908-NEXT: v_perm_b32 v7, v7, v6, s4
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX908-NEXT: s_movk_i32 s14, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX908-NEXT: s_mov_b32 s15, 0x7060302
; GFX908-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Loop Header: Depth=1
; GFX908-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX908-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX908-NEXT: v_min_f32_e32 v6, v7, v6
-; GFX908-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX908-NEXT: v_add3_u32 v7, v7, v6, s14
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v8
+; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX908-NEXT: v_add3_u32 v6, v6, v5, s14
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v9
+; GFX908-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX908-NEXT: v_add3_u32 v10, v10, v6, s14
+; GFX908-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX908-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX908-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX908-NEXT: v_min_f32_e32 v7, v9, v7
-; GFX908-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX908-NEXT: v_add3_u32 v9, v9, v7, s14
-; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX908-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX908-NEXT: v_perm_b32 v7, v7, v6, s15
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX908-NEXT: v_perm_b32 v6, v6, v5, s15
; GFX908-NEXT: s_mov_b64 s[12:13], exec
+; GFX908-NEXT: v_mov_b32_e32 v5, v6
; GFX908-NEXT: v_mov_b32_e32 v6, v7
-; GFX908-NEXT: v_mov_b32_e32 v7, v8
; GFX908-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
; GFX908-NEXT: v_readfirstlane_b32 s8, v0
@@ -9294,21 +9224,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX908-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX908-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX908-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_6
; GFX908-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX908-NEXT: s_mov_b64 exec, s[12:13]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v8, v6
+; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: buffer_wbinvl1
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB21_5
; GFX908-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v0, v6
+; GFX908-NEXT: v_mov_b32_e32 v0, v5
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9348,34 +9278,34 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_mov_b64 exec, s[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_e32 v8, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v7, v6, v7
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
; GFX8-NEXT: .LBB21_5: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Loop Header: Depth=1
; GFX8-NEXT: ; Child Loop BB21_6 Depth 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v8
-; GFX8-NEXT: v_min_f32_e32 v6, v7, v6
-; GFX8-NEXT: v_bfe_u32 v7, v6, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v6
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v7
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v8
+; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v9
+; GFX8-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, v10, v6
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, 0x7fff, v10
+; GFX8-NEXT: v_or_b32_e32 v11, 0x400000, v6
; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX8-NEXT: v_and_b32_e32 v9, 0xffff0000, v8
-; GFX8-NEXT: v_min_f32_e32 v7, v9, v7
-; GFX8-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v7
-; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
-; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v7, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_alignbit_b32 v7, v7, v6, 16
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v6, v6, v5, 16
; GFX8-NEXT: s_mov_b64 s[12:13], exec
+; GFX8-NEXT: v_mov_b32_e32 v5, v6
; GFX8-NEXT: v_mov_b32_e32 v6, v7
-; GFX8-NEXT: v_mov_b32_e32 v7, v8
; GFX8-NEXT: .LBB21_6: ; Parent Loop BB21_5 Depth=1
; GFX8-NEXT: ; => This Inner Loop Header: Depth=2
; GFX8-NEXT: v_readfirstlane_b32 s8, v0
@@ -9387,21 +9317,21 @@ define <2 x bfloat> @buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterf
; GFX8-NEXT: s_and_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_atomic_cmpswap v[6:7], v4, s[8:11], 0 offen offset:1024 glc
+; GFX8-NEXT: buffer_atomic_cmpswap v[5:6], v4, s[8:11], 0 offen offset:1024 glc
; GFX8-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_6
; GFX8-NEXT: ; %bb.7: ; in Loop: Header=BB21_5 Depth=1
; GFX8-NEXT: s_mov_b64 exec, s[12:13]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v6, v8
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v8, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: buffer_wbinvl1
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB21_5
; GFX8-NEXT: ; %bb.8: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v0, v6
+; GFX8-NEXT: v_mov_b32_e32 v0, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_v2bf16__offset__waterfall__amdgpu_no_fine_grained_memory:
@@ -9617,18 +9547,18 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX942-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, v0
+; GFX942-NEXT: v_mov_b32_e32 v1, v0
; GFX942-NEXT: v_mov_b32_e32 v0, s16
; GFX942-NEXT: buffer_load_dword v0, v0, s[0:3], 0 offen offset:1024
; GFX942-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
; GFX942-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v5, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_min_f32_e32 v4, v0, v2
; GFX942-NEXT: v_mov_b64_e32 v[0:1], v[4:5]
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
@@ -9668,18 +9598,18 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX90A-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v2, v0
+; GFX90A-NEXT: v_mov_b32_e32 v1, v0
; GFX90A-NEXT: v_mov_b32_e32 v0, s20
; GFX90A-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX90A-NEXT: v_mov_b32_e32 v3, s20
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_mov_b32_e32 v5, v0
-; GFX90A-NEXT: v_max_f32_e32 v1, v2, v2
; GFX90A-NEXT: v_max_f32_e32 v0, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v0, v1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s20
+; GFX90A-NEXT: v_min_f32_e32 v4, v0, v2
; GFX90A-NEXT: v_pk_mov_b32 v[0:1], v[4:5], v[4:5] op_sel:[0,1]
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
@@ -9698,24 +9628,24 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX908-LABEL: buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v2, v0
+; GFX908-NEXT: v_mov_b32_e32 v1, v0
; GFX908-NEXT: v_mov_b32_e32 v0, s20
; GFX908-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v1, v1
+; GFX908-NEXT: v_mov_b32_e32 v3, s20
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v0, v1
-; GFX908-NEXT: v_mov_b32_e32 v5, s20
-; GFX908-NEXT: v_mov_b32_e32 v0, v3
-; GFX908-NEXT: v_mov_b32_e32 v1, v4
-; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX908-NEXT: v_mov_b32_e32 v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v5, v5
+; GFX908-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v0, v4
+; GFX908-NEXT: v_mov_b32_e32 v1, v5
+; GFX908-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_1
@@ -9731,19 +9661,19 @@ define float @buffer_fat_ptr_system_atomic_fmin_ret_f32__offset__amdgpu_no_fine_
; GFX8-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: v_mul_f32_e32 v2, 1.0, v1
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v0, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: v_mov_b32_e32 v1, v4
-; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v0
+; GFX8-NEXT: v_mul_f32_e32 v0, 1.0, v5
+; GFX8-NEXT: v_min_f32_e32 v4, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, v4
+; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
index d2c22b4689dae..f19dc587ca65b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_cond.ll
@@ -25,26 +25,26 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, v4, v2
-; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v0, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
+; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -76,6 +76,7 @@ define void @buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s16
+; GFX11-NEXT: v_mov_b32_e32 v3, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB1_1: ; %atomicrmw.start
@@ -84,16 +85,15 @@ define void @buffer_fat_ptr_agent_atomic_usub_cond_noret_u32__offset__amdgpu_no_
; GFX11-NEXT: v_sub_nc_u32_e32 v1, v2, v0
; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v2, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
-; GFX11-NEXT: v_dual_mov_b32 v5, s16 :: v_dual_mov_b32 v4, v2
-; GFX11-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[3:4], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v2
+; GFX11-NEXT: v_mov_b32_e32 v2, v4
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -124,26 +124,26 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_rem
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, v4, v2
-; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v0, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
+; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -174,26 +174,26 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fin
; GFX11-LABEL: buffer_fat_ptr_agent_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, v4, v2
-; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v0, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
+; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -229,26 +229,26 @@ define i32 @buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fi
; GFX11-LABEL: buffer_fat_ptr_system_atomic_usub_cond_ret_u32__offset__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, v0 :: v_dual_mov_b32 v3, s16
; GFX11-NEXT: v_mov_b32_e32 v0, s16
; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[0:3], 0 offen offset:1024
; GFX11-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, s16
+; GFX11-NEXT: v_mov_b32_e32 v5, v0
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, v4, v2
-; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v4, v2
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v0, vcc_lo
+; GFX11-NEXT: v_sub_nc_u32_e32 v0, v5, v2
+; GFX11-NEXT: v_cmp_ge_u32_e32 vcc_lo, v5, v2
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v5, v0, vcc_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, v4 :: v_dual_mov_b32 v0, v3
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v5, s[0:3], 0 offen offset:1024 glc
+; GFX11-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v0, v4
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[0:1], v3, s[0:3], 0 offen offset:1024 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
index 5224090063464..0cd34eff9b269 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-usub_sat.ll
@@ -40,18 +40,18 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX9-NEXT: v_mov_b32_e32 v0, s20
; GFX9-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s20
; GFX9-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
-; GFX9-NEXT: v_mov_b32_e32 v5, s20
-; GFX9-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: v_sub_u32_e64 v4, v5, v2 clamp
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB0_1
@@ -95,19 +95,19 @@ define void @buffer_fat_ptr_agent_atomic_usub_sat_noret_u32__offset__amdgpu_no_f
; GFX9-NEXT: v_mov_b32_e32 v1, s20
; GFX9-NEXT: buffer_load_dword v2, v1, s[16:19], 0 offen offset:1024
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s20
; GFX9-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_sub_u32_e64 v1, v2, v0 clamp
-; GFX9-NEXT: v_mov_b32_e32 v5, s20
-; GFX9-NEXT: v_mov_b32_e32 v4, v2
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: buffer_atomic_cmpswap v[3:4], v5, s[16:19], 0 offen offset:1024 glc
+; GFX9-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-NEXT: v_mov_b32_e32 v4, v1
+; GFX9-NEXT: buffer_atomic_cmpswap v[4:5], v3, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v4, v2
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX9-NEXT: v_mov_b32_e32 v2, v3
+; GFX9-NEXT: v_mov_b32_e32 v2, v4
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -151,18 +151,18 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_remo
; GFX9-NEXT: v_mov_b32_e32 v0, s20
; GFX9-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s20
; GFX9-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
-; GFX9-NEXT: v_mov_b32_e32 v5, s20
-; GFX9-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: v_sub_u32_e64 v4, v5, v2 clamp
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB2_1
@@ -207,18 +207,18 @@ define i32 @buffer_fat_ptr_agent_atomic_usub_sat_ret_u32__offset__amdgpu_no_fine
; GFX9-NEXT: v_mov_b32_e32 v0, s20
; GFX9-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s20
; GFX9-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
-; GFX9-NEXT: v_mov_b32_e32 v5, s20
-; GFX9-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: v_sub_u32_e64 v4, v5, v2 clamp
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB3_1
@@ -268,18 +268,18 @@ define i32 @buffer_fat_ptr_system_atomic_usub_sat_ret_u32__offset__amdgpu_no_fin
; GFX9-NEXT: v_mov_b32_e32 v0, s20
; GFX9-NEXT: buffer_load_dword v0, v0, s[16:19], 0 offen offset:1024
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s20
; GFX9-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, v0
-; GFX9-NEXT: v_sub_u32_e64 v3, v4, v2 clamp
-; GFX9-NEXT: v_mov_b32_e32 v5, s20
-; GFX9-NEXT: v_mov_b32_e32 v0, v3
-; GFX9-NEXT: v_mov_b32_e32 v1, v4
-; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v5, s[16:19], 0 offen offset:1024 glc
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: v_sub_u32_e64 v4, v5, v2 clamp
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: v_mov_b32_e32 v1, v5
+; GFX9-NEXT: buffer_atomic_cmpswap v[0:1], v3, s[16:19], 0 offen offset:1024 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB4_1
diff --git a/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll b/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
index 5ce7549866971..c87a7fba94ab3 100644
--- a/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
+++ b/llvm/test/CodeGen/AMDGPU/codegen-prepare-addrspacecast-non-null.ll
@@ -361,20 +361,20 @@ define void @recursive_phis(i1 %cond, ptr addrspace(5) %ptr) {
; GISEL-ASM-NEXT: v_and_b32_e32 v0, 0xffff, v1
; GISEL-ASM-NEXT: ; %bb.2: ; %finallyendcf.split
; GISEL-ASM-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-ASM-NEXT: s_mov_b64 s[6:7], src_private_base
-; GISEL-ASM-NEXT: s_mov_b64 s[8:9], 0
+; GISEL-ASM-NEXT: s_mov_b64 s[8:9], src_private_base
+; GISEL-ASM-NEXT: s_mov_b64 s[6:7], 0
+; GISEL-ASM-NEXT: v_mov_b32_e32 v1, s9
; GISEL-ASM-NEXT: v_mov_b32_e32 v2, 7
; GISEL-ASM-NEXT: .LBB11_3: ; %finally
; GISEL-ASM-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-ASM-NEXT: s_and_b64 s[10:11], exec, s[4:5]
-; GISEL-ASM-NEXT: s_or_b64 s[8:9], s[10:11], s[8:9]
-; GISEL-ASM-NEXT: v_mov_b32_e32 v1, s7
+; GISEL-ASM-NEXT: s_and_b64 s[8:9], exec, s[4:5]
+; GISEL-ASM-NEXT: s_or_b64 s[6:7], s[8:9], s[6:7]
; GISEL-ASM-NEXT: flat_store_dword v[0:1], v2
; GISEL-ASM-NEXT: s_waitcnt vmcnt(0)
-; GISEL-ASM-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GISEL-ASM-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GISEL-ASM-NEXT: s_cbranch_execnz .LBB11_3
; GISEL-ASM-NEXT: ; %bb.4: ; %end
-; GISEL-ASM-NEXT: s_or_b64 exec, exec, s[8:9]
+; GISEL-ASM-NEXT: s_or_b64 exec, exec, s[6:7]
; GISEL-ASM-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-ASM-NEXT: s_setpc_b64 s[30:31]
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 7d36cc7819330..0a6dbff0e55f1 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -17641,43 +17641,44 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17692,39 +17693,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17739,34 +17742,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB68_1
@@ -17778,41 +17781,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX90A-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB68_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -17820,33 +17823,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB68_1
@@ -17860,34 +17863,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB68_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB68_1
@@ -17970,43 +17973,44 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18021,39 +18025,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18069,35 +18075,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB69_1
@@ -18108,41 +18114,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX90A-LABEL: flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB69_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18150,33 +18156,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB69_1
@@ -18192,34 +18198,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB69_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB69_1
@@ -18309,6 +18315,8 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -18316,36 +18324,36 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_and_b32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, v5, v0 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18361,6 +18369,8 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -18368,33 +18378,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_lshlrev_b32 v0, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v0, v6, v0 :: v_dual_and_b32 v7, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18409,35 +18419,35 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB70_1
@@ -18455,28 +18465,28 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_add_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -18499,33 +18509,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v0, s9
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB70_1
@@ -18540,34 +18550,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB70_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB70_1
@@ -18651,44 +18661,44 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18701,41 +18711,41 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18748,35 +18758,35 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX10-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB71_1
@@ -18787,36 +18797,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX90A-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB71_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18826,36 +18836,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX908-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB71_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18865,37 +18875,37 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX8-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB71_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB71_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18971,44 +18981,44 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -19021,41 +19031,41 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -19070,35 +19080,35 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB72_1
@@ -19109,36 +19119,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB72_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19148,36 +19158,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB72_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19189,37 +19199,37 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB72_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB72_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19304,44 +19314,44 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -19357,41 +19367,41 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -19406,35 +19416,35 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB73_1
@@ -19452,28 +19462,28 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_add_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_add_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -19496,28 +19506,28 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_add_f32_e32 v0, v5, v0
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX908-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -19535,37 +19545,37 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB73_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB73_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19646,43 +19656,44 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -19697,39 +19708,41 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -19745,35 +19758,35 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_add_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB74_1
@@ -19784,44 +19797,44 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX90A-LABEL: flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB74_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -19829,33 +19842,33 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB74_1
@@ -19871,34 +19884,34 @@ define <2 x bfloat> @flat_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB74_1
@@ -19983,44 +19996,44 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -20033,41 +20046,41 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -20082,35 +20095,35 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB75_1
@@ -20121,39 +20134,39 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX90A-LABEL: flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB75_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20163,36 +20176,36 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX908-LABEL: flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB75_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20204,37 +20217,37 @@ define void @flat_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB75_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20314,43 +20327,44 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -20365,39 +20379,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -20412,34 +20428,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB76_1
@@ -20451,41 +20467,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX90A-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB76_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
@@ -20493,33 +20509,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB76_1
@@ -20533,34 +20549,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory(
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB76_1
@@ -20642,44 +20658,44 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -20692,41 +20708,41 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -20739,35 +20755,35 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX10-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB77_1
@@ -20778,36 +20794,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX90A-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB77_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20817,36 +20833,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX908-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB77_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20856,37 +20872,37 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr %p
; GFX8-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB77_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -20963,43 +20979,44 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -21014,39 +21031,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -21061,34 +21080,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB78_1
@@ -21100,41 +21119,41 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX90A-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB78_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -21142,33 +21161,33 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB78_1
@@ -21182,34 +21201,34 @@ define <2 x bfloat> @flat_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB78_1
@@ -21291,44 +21310,44 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -21341,41 +21360,41 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -21388,35 +21407,35 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX10-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB79_1
@@ -21427,36 +21446,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX90A-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB79_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -21466,36 +21485,36 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX908-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB79_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -21505,37 +21524,37 @@ define void @flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory_
; GFX8-LABEL: flat_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB79_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 61bc7c72259d8..91d75d7077259 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -31,27 +31,27 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB0_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -77,25 +77,25 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -103,13 +103,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -174,27 +174,27 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -222,25 +222,25 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -248,13 +248,13 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -327,24 +327,24 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v1, v7, v7
-; GFX942-NEXT: v_max_f32_e32 v6, v1, v0
+; GFX942-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -386,12 +386,12 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -414,17 +414,17 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_1
@@ -488,22 +488,22 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB3_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -535,20 +535,20 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -558,20 +558,20 @@ define void @flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -629,22 +629,22 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -678,20 +678,20 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -701,20 +701,20 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -781,24 +781,24 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -842,12 +842,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -870,12 +870,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v1, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -945,27 +945,27 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX942-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB6_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -993,28 +993,28 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1022,13 +1022,13 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1098,22 +1098,22 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX942-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1147,23 +1147,23 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1173,20 +1173,20 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1249,41 +1249,42 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_max_f32 v3, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX11-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1303,14 +1304,14 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1328,25 +1329,25 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
@@ -1354,13 +1355,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1441,27 +1442,27 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1487,25 +1488,25 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1513,13 +1514,13 @@ define float @flat_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1588,27 +1589,27 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1634,25 +1635,25 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1660,13 +1661,13 @@ define float @flat_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1731,27 +1732,27 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1779,25 +1780,25 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1805,13 +1806,13 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1884,24 +1885,24 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v1, v7, v7
-; GFX942-NEXT: v_max_f32_e32 v6, v1, v0
+; GFX942-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1943,12 +1944,12 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -1971,17 +1972,17 @@ define float @flat_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_max_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
@@ -2045,22 +2046,22 @@ define void @flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2092,20 +2093,20 @@ define void @flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2115,20 +2116,20 @@ define void @flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2186,22 +2187,22 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB14_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2235,20 +2236,20 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2258,20 +2259,20 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2338,24 +2339,24 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2399,12 +2400,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -2427,12 +2428,12 @@ define void @flat_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v1, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2502,27 +2503,27 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX942-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB16_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2550,28 +2551,28 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX90A-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2579,13 +2580,13 @@ define float @flat_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2655,22 +2656,22 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX942-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2704,23 +2705,23 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX90A-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2730,20 +2731,20 @@ define void @flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX908-LABEL: flat_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2801,29 +2802,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2832,26 +2833,25 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB18_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2898,29 +2898,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB18_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -2928,23 +2928,22 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB18_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB18_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3034,95 +3033,90 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_6
-; GFX908-NEXT: .LBB18_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB18_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB18_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB18_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB18_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB18_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB18_2
-; GFX908-NEXT: .LBB18_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB18_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB18_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB18_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB18_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB18_6
@@ -3130,18 +3124,17 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -3199,6 +3192,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3224,10 +3218,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[8:9]
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[6:7], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3246,7 +3239,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execz .LBB19_2
; GFX12-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3303,6 +3295,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3325,10 +3318,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
-; GFX11-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3347,7 +3339,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: s_cbranch_execz .LBB19_2
; GFX11-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX11-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3445,6 +3436,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7f8, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
@@ -3467,9 +3459,8 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3485,7 +3476,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_cbranch_execz .LBB19_2
; GFX908-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
@@ -3501,6 +3491,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
@@ -3528,9 +3519,8 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3546,7 +3536,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_cbranch_execz .LBB19_2
; GFX8-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
; GFX8-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
@@ -3615,6 +3604,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3640,10 +3630,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[8:9]
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[6:7], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3662,7 +3651,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execz .LBB20_2
; GFX12-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3720,6 +3708,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3742,10 +3731,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
-; GFX11-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3764,7 +3752,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: s_cbranch_execz .LBB20_2
; GFX11-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX11-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3862,6 +3849,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
@@ -3884,9 +3872,8 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3902,7 +3889,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_cbranch_execz .LBB20_2
; GFX908-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
@@ -3918,6 +3904,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -3945,9 +3932,8 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3963,7 +3949,6 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_cbranch_execz .LBB20_2
; GFX8-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
; GFX8-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
@@ -4032,6 +4017,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4047,21 +4033,20 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4070,20 +4055,19 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB21_2
; GFX12-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[6:7]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4129,6 +4113,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
@@ -4141,22 +4126,21 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -4164,19 +4148,18 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB21_2
; GFX11-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -4262,6 +4245,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -4274,40 +4258,38 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB21_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB21_2
; GFX908-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -4315,6 +4297,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -4329,44 +4312,42 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB21_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB21_2
; GFX8-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -4422,13 +4403,14 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB22_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4440,21 +4422,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4462,21 +4443,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB22_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB22_2
; GFX12-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4524,12 +4504,13 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB22_3
; GFX11-NEXT: ; %bb.1: ; %Flow2
@@ -4539,42 +4520,40 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB22_4
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB22_2
; GFX11-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -4664,10 +4643,11 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0x7f8, v0
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0x7f8, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_3
@@ -4678,40 +4658,38 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB22_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB22_2
; GFX908-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -4719,12 +4697,13 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v7
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_3
@@ -4735,44 +4714,42 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v2, v[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB22_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB22_2
; GFX8-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -4831,13 +4808,14 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB23_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4849,21 +4827,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4871,21 +4848,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB23_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB23_2
; GFX12-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4934,12 +4910,13 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB23_3
; GFX11-NEXT: ; %bb.1: ; %Flow2
@@ -4949,42 +4926,40 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB23_4
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB23_2
; GFX11-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -5074,10 +5049,11 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: v_addc_co_u32_e32 v7, vcc, -1, v1, vcc
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB23_3
@@ -5088,40 +5064,38 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB23_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB23_2
; GFX908-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -5129,12 +5103,13 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v7
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_3
@@ -5145,44 +5120,42 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v2, v[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB23_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB23_2
; GFX8-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -5241,29 +5214,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5272,26 +5245,25 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB24_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
@@ -5338,29 +5310,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB24_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -5368,78 +5340,76 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB24_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB24_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX10-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX10-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v1
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX10-NEXT: s_cbranch_execz .LBB24_4
; GFX10-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX10-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX10-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v7, v5
-; GFX10-NEXT: v_mov_b32_e32 v6, v4
-; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX10-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX10-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX10-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX10-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX10-NEXT: .LBB24_4: ; %Flow2
; GFX10-NEXT: s_andn2_saveexec_b32 s4, s4
; GFX10-NEXT: s_cbranch_execz .LBB24_6
; GFX10-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX10-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX10-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX10-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; GFX10-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX10-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX10-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
+; GFX10-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX10-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX10-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX10-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
; GFX10-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_mov_b32_e32 v0, v4
-; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
@@ -5447,144 +5417,138 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB24_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX90A-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX90A-NEXT: .LBB24_4: ; %Flow2
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB24_6
; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
-; GFX90A-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
-; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX90A-NEXT: v_max_f64 v[0:1], v[0:1], v[6:7]
+; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v4
-; GFX90A-NEXT: v_mov_b32_e32 v1, v5
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_6
-; GFX908-NEXT: .LBB24_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB24_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB24_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB24_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB24_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB24_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB24_2
-; GFX908-NEXT: .LBB24_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB24_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB24_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB24_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB24_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB24_6
@@ -5592,18 +5556,17 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -5612,37 +5575,37 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX7-NEXT: s_cbranch_execz .LBB24_4
; GFX7-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
-; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v5, v[4:5]
-; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v3, v[2:3]
+; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX7-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX7-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX7-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB24_2
; GFX7-NEXT: ; %bb.3: ; %Flow
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: .LBB24_4: ; %Flow2
; GFX7-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_cbranch_execz .LBB24_6
@@ -5650,18 +5613,17 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX7-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX7-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX7-NEXT: v_add_i32_e32 v7, vcc, 4, v6
-; GFX7-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX7-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX7-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX7-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX7-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
+; GFX7-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX7-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX7-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX7-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v0, v4
-; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmax ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -5676,29 +5638,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5707,26 +5669,25 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB25_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5773,29 +5734,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX11-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB25_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -5803,23 +5764,22 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB25_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB25_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5909,95 +5869,90 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_6
-; GFX908-NEXT: .LBB25_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB25_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB25_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB25_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB25_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB25_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB25_2
-; GFX908-NEXT: .LBB25_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB25_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB25_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB25_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB25_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB25_6
@@ -6005,18 +5960,17 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -6079,8 +6033,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6092,12 +6047,11 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6125,8 +6079,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6138,12 +6093,11 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6175,13 +6129,13 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -6203,8 +6157,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6216,12 +6171,11 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6245,8 +6199,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6258,12 +6213,11 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6287,6 +6241,7 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -6298,10 +6253,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6330,13 +6284,13 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
@@ -6364,14 +6318,14 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -6398,17 +6352,17 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6467,33 +6421,35 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -6504,7 +6460,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6517,24 +6473,25 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6561,67 +6518,69 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -6632,18 +6591,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6653,12 +6613,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6683,9 +6642,10 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -6694,10 +6654,9 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6717,36 +6676,36 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6762,14 +6721,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -6797,17 +6756,17 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6868,33 +6827,35 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -6905,7 +6866,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6918,24 +6879,25 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6963,67 +6925,69 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -7034,18 +6998,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7055,12 +7020,11 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7085,9 +7049,10 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -7096,10 +7061,9 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7119,36 +7083,36 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7164,14 +7128,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -7199,17 +7163,17 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -7271,8 +7235,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7284,10 +7249,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7316,8 +7280,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7329,10 +7294,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7365,13 +7329,13 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -7392,8 +7356,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7405,10 +7370,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7433,8 +7397,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7446,10 +7411,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7474,6 +7438,7 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -7485,9 +7450,8 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX10-NEXT: v_max_f16_e32 v3, v3, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7516,13 +7480,13 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
@@ -7549,13 +7513,13 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
@@ -7582,16 +7546,16 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
+; GFX8-NEXT: v_max_f16_e32 v3, v3, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -7649,36 +7613,38 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7695,36 +7661,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7741,30 +7708,30 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB30_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7774,37 +7741,39 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7816,37 +7785,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7858,31 +7827,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB30_1
@@ -7893,31 +7862,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7927,31 +7896,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7961,32 +7930,32 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8040,36 +8009,38 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8086,36 +8057,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8133,30 +8105,30 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB31_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8166,37 +8138,39 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8208,37 +8182,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8250,31 +8224,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB31_1
@@ -8285,31 +8259,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8319,31 +8293,31 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8353,32 +8327,32 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8433,15 +8407,15 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8468,15 +8442,15 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8498,44 +8472,44 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB32_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8558,15 +8532,15 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8590,23 +8564,23 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX10-NEXT: v_max_f16_e32 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX10-NEXT: v_max_f16_e32 v0, v0, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX10-NEXT: v_and_or_b32 v5, 0xffff0000, v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB32_1
@@ -8617,27 +8591,27 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -8645,14 +8619,14 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -8673,19 +8647,19 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v5, v2, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
@@ -8734,14 +8708,14 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8767,24 +8741,24 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8797,24 +8771,24 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX942-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB33_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8825,14 +8799,14 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8854,25 +8828,25 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8886,23 +8860,23 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB33_1
@@ -8913,22 +8887,22 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8938,22 +8912,22 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX908-LABEL: flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2046
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f16_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2046 glc
+; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX908-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8965,22 +8939,22 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9027,34 +9001,36 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -9065,7 +9041,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9078,24 +9054,25 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -9123,67 +9100,69 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB34_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -9194,18 +9173,19 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9215,12 +9195,11 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -9245,9 +9224,10 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -9256,10 +9236,9 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9279,39 +9258,39 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX90A-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9327,14 +9306,14 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -9362,17 +9341,17 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -9433,37 +9412,39 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9480,37 +9461,38 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9527,30 +9509,30 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB35_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9560,37 +9542,39 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9602,37 +9586,37 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9644,31 +9628,31 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB35_1
@@ -9679,34 +9663,34 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9716,31 +9700,31 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX908-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9750,32 +9734,32 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX8-LABEL: flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13287,15 +13271,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13314,43 +13298,43 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX942-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB46_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13370,14 +13354,14 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13395,25 +13379,25 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -13421,13 +13405,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13445,21 +13429,21 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_max_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
@@ -13518,15 +13502,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13545,43 +13529,43 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX942-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB47_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13602,21 +13586,21 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_max_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB47_1
@@ -13627,25 +13611,25 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13653,13 +13637,13 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13679,21 +13663,21 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
@@ -13754,15 +13738,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13785,25 +13769,25 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v2, v2
; GFX942-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v1, v7, v7
+; GFX942-NEXT: v_pk_max_f16 v0, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v6, v1, v0
+; GFX942-NEXT: v_pk_max_f16 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB48_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13816,22 +13800,22 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_pk_max_f16 v1, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX11-NEXT: v_pk_max_f16 v0, v5, v0
+; GFX11-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX11-NEXT: v_pk_max_f16 v5, v0, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13845,21 +13829,21 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_max_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB48_1
@@ -13877,12 +13861,12 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -13905,17 +13889,17 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX908-NEXT: v_pk_max_f16 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB48_1
@@ -13930,21 +13914,21 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB48_1
@@ -14004,21 +13988,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14031,23 +14015,23 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB49_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14057,22 +14041,22 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14084,21 +14068,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB49_1
@@ -14109,20 +14093,20 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14132,20 +14116,20 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14155,24 +14139,24 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14225,21 +14209,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14252,23 +14236,23 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14278,22 +14262,22 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14307,21 +14291,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -14332,20 +14316,20 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14355,20 +14339,20 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14380,24 +14364,24 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14453,21 +14437,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14484,25 +14468,25 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14515,22 +14499,22 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14544,21 +14528,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -14576,12 +14560,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_max_f16 v0, v3, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_max_f16 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -14604,12 +14588,12 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v0, v5, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX908-NEXT: v_pk_max_f16 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14627,24 +14611,24 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14701,15 +14685,15 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14729,43 +14713,43 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX942-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14786,21 +14770,21 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_max_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB52_1
@@ -14811,28 +14795,28 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14840,13 +14824,13 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14866,21 +14850,21 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -14940,22 +14924,22 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14968,23 +14952,23 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14994,22 +14978,22 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX11-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15023,21 +15007,21 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -15048,23 +15032,23 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15074,20 +15058,20 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15099,24 +15083,24 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15177,41 +15161,43 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15230,38 +15216,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15275,87 +15262,88 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX942-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15370,39 +15358,41 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15417,34 +15407,34 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -15456,41 +15446,41 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15498,33 +15488,33 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
@@ -15538,34 +15528,34 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
@@ -15629,41 +15619,43 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15682,38 +15674,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15727,87 +15720,88 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX942-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15822,39 +15816,41 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15870,35 +15866,35 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -15909,41 +15905,41 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX90A-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15951,33 +15947,33 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB55_1
@@ -15993,34 +15989,34 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
@@ -16086,41 +16082,43 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16139,38 +16137,39 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16188,41 +16187,41 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v7
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX942-NEXT: v_max_f32_e32 v0, v1, v0
-; GFX942-NEXT: v_max_f32_e32 v1, v6, v3
-; GFX942-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX942-NEXT: v_add3_u32 v3, v3, v0, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v1, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX942-NEXT: v_add3_u32 v7, v7, v0, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v2, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v6, v1, v0, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v2, v0, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16235,6 +16234,8 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -16242,36 +16243,36 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_and_b32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_max_f32 v0, v5, v0 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16287,6 +16288,8 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -16294,33 +16297,33 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_lshlrev_b32 v0, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v0, v6, v0 :: v_dual_and_b32 v7, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16335,35 +16338,35 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -16381,28 +16384,28 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -16425,33 +16428,33 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v0, s9
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -16466,34 +16469,34 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -16558,43 +16561,42 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16611,40 +16613,40 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16657,39 +16659,39 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX942-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16699,44 +16701,44 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16749,41 +16751,41 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16796,35 +16798,35 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX10-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -16835,36 +16837,36 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16874,36 +16876,36 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16913,37 +16915,37 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX8-LABEL: flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17000,43 +17002,42 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17053,40 +17054,40 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17099,39 +17100,39 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB58_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17141,44 +17142,44 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17191,41 +17192,41 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17240,35 +17241,35 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB58_1
@@ -17279,36 +17280,36 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17318,36 +17319,36 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17359,37 +17360,37 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17449,43 +17450,42 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17502,40 +17502,40 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17552,41 +17552,41 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB59_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17599,44 +17599,44 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17652,41 +17652,41 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17701,35 +17701,35 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB59_1
@@ -17747,28 +17747,28 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_max_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_max_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -17791,28 +17791,28 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_max_f32_e32 v0, v5, v0
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX908-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -17830,37 +17830,37 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17921,42 +17921,44 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17975,39 +17977,40 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18021,87 +18024,88 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX942-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB60_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18116,39 +18120,41 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18164,35 +18170,35 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_max_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB60_1
@@ -18203,44 +18209,44 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX90A-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18248,33 +18254,33 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB60_1
@@ -18290,34 +18296,34 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB60_1
@@ -18382,44 +18388,43 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18436,41 +18441,41 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18483,39 +18488,39 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX942-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB61_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18525,44 +18530,44 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18575,41 +18580,41 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18624,35 +18629,35 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB61_1
@@ -18663,39 +18668,39 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX90A-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18705,36 +18710,36 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX908-LABEL: flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18746,37 +18751,37 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index a0f4e7ca39244..f5564f14eee24 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -31,27 +31,27 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB0_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -77,25 +77,25 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -103,13 +103,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -174,27 +174,27 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -222,25 +222,25 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -248,13 +248,13 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grai
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -327,24 +327,24 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v1, v7, v7
-; GFX942-NEXT: v_min_f32_e32 v6, v1, v0
+; GFX942-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -386,12 +386,12 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -414,17 +414,17 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grai
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB2_1
@@ -488,22 +488,22 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB3_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -535,20 +535,20 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -558,20 +558,20 @@ define void @flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -629,22 +629,22 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -678,20 +678,20 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -701,20 +701,20 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -781,24 +781,24 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -842,12 +842,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -870,12 +870,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v1, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -945,27 +945,27 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX942-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB6_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -993,28 +993,28 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1022,13 +1022,13 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1098,22 +1098,22 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX942-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1147,23 +1147,23 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1173,20 +1173,20 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1249,41 +1249,42 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_max_f32 v3, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX11-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1303,14 +1304,14 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -1328,25 +1329,25 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
@@ -1354,13 +1355,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1441,27 +1442,27 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1487,25 +1488,25 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1513,13 +1514,13 @@ define float @flat_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amd
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1588,27 +1589,27 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1634,25 +1635,25 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1660,13 +1661,13 @@ define float @flat_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1731,27 +1732,27 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1779,25 +1780,25 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1805,13 +1806,13 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1884,24 +1885,24 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v1, v2, v2
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v1, v7, v7
-; GFX942-NEXT: v_min_f32_e32 v6, v1, v0
+; GFX942-NEXT: v_max_f32_e32 v0, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1943,12 +1944,12 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -1971,17 +1972,17 @@ define float @flat_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v1, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v6, v6
+; GFX908-NEXT: v_min_f32_e32 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB12_1
@@ -2045,22 +2046,22 @@ define void @flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2092,20 +2093,20 @@ define void @flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2115,20 +2116,20 @@ define void @flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memor
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2186,22 +2187,22 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB14_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2235,20 +2236,20 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2258,20 +2259,20 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2338,24 +2339,24 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2399,12 +2400,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v3, v1, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
+; GFX90A-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -2427,12 +2428,12 @@ define void @flat_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fin
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v1, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v5, v0
+; GFX908-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2502,27 +2503,27 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX942-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB16_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2550,28 +2551,28 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX90A-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2579,13 +2580,13 @@ define float @flat_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fin
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2655,22 +2656,22 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX942-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2704,23 +2705,23 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX90A-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2730,20 +2731,20 @@ define void @flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX908-LABEL: flat_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2801,29 +2802,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2832,26 +2833,25 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB18_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -2898,29 +2898,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB18_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -2928,23 +2928,22 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB18_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB18_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3034,95 +3033,90 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_6
-; GFX908-NEXT: .LBB18_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB18_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB18_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB18_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB18_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB18_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB18_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB18_2
-; GFX908-NEXT: .LBB18_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB18_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB18_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB18_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB18_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB18_6
@@ -3130,18 +3124,17 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -3199,6 +3192,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3224,10 +3218,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[8:9]
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[6:7], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3246,7 +3239,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execz .LBB19_2
; GFX12-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3303,6 +3295,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3325,10 +3318,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
-; GFX11-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX11-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3347,7 +3339,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: s_cbranch_execz .LBB19_2
; GFX11-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX11-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3445,6 +3436,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7f8, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
@@ -3467,9 +3459,8 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3485,7 +3476,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_cbranch_execz .LBB19_2
; GFX908-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
@@ -3501,6 +3491,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
@@ -3528,9 +3519,8 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3546,7 +3536,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_cbranch_execz .LBB19_2
; GFX8-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
; GFX8-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
@@ -3615,6 +3604,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3640,10 +3630,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[8:9]
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[6:7], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3662,7 +3651,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execz .LBB20_2
; GFX12-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
@@ -3720,6 +3708,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3742,10 +3731,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
-; GFX11-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX11-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -3764,7 +3752,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: s_cbranch_execz .LBB20_2
; GFX11-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
; GFX11-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3862,6 +3849,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
@@ -3884,9 +3872,8 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v9, v1
; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3902,7 +3889,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_cbranch_execz .LBB20_2
; GFX908-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
@@ -3918,6 +3904,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
@@ -3945,9 +3932,8 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3963,7 +3949,6 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_cbranch_execz .LBB20_2
; GFX8-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
; GFX8-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
@@ -4032,6 +4017,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4047,21 +4033,20 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4070,20 +4055,19 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB21_2
; GFX12-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[6:7]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4129,6 +4113,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
@@ -4141,22 +4126,21 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -4164,19 +4148,18 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB21_2
; GFX11-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -4262,6 +4245,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
@@ -4274,40 +4258,38 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB21_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB21_2
; GFX908-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -4315,6 +4297,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
@@ -4329,44 +4312,42 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v5, v[2:3]
+; GFX8-NEXT: flat_load_dword v4, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB21_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB21_2
; GFX8-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -4422,13 +4403,14 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB22_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4440,21 +4422,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4462,21 +4443,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB22_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB22_2
; GFX12-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4524,12 +4504,13 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB22_3
; GFX11-NEXT: ; %bb.1: ; %Flow2
@@ -4539,42 +4520,40 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB22_4
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB22_2
; GFX11-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -4664,10 +4643,11 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0x7f8, v0
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0x7f8, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v1, vcc
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_3
@@ -4678,40 +4658,38 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB22_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB22_2
; GFX908-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -4719,12 +4697,13 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7f8, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v7
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_3
@@ -4735,44 +4714,42 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v2, v[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB22_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB22_2
; GFX8-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -4831,13 +4808,14 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB23_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4849,21 +4827,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4871,21 +4848,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB23_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB23_2
; GFX12-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4934,12 +4910,13 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX11-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execnz .LBB23_3
; GFX11-NEXT: ; %bb.1: ; %Flow2
@@ -4949,42 +4926,40 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
; GFX11-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: s_cbranch_execnz .LBB23_4
; GFX11-NEXT: ; %bb.5: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB23_2
; GFX11-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX11-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
+; GFX11-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX11-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX11-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX11-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
@@ -5074,10 +5049,11 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX908-NEXT: v_add_co_u32_e32 v6, vcc, 0xfffff800, v0
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: v_addc_co_u32_e32 v7, vcc, -1, v1, vcc
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v7
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB23_3
@@ -5088,40 +5064,38 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_setpc_b64 s[30:31]
; GFX908-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[6:7]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
; GFX908-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX908-NEXT: v_mov_b32_e32 v3, v1
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v2, v0
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB23_4
; GFX908-NEXT: ; %bb.5: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX908-NEXT: s_cbranch_execz .LBB23_2
; GFX908-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX908-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX908-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX908-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX908-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
@@ -5129,12 +5103,13 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, -1, v1, vcc
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
+; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v7
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_3
@@ -5145,44 +5120,42 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_setpc_b64 s[30:31]
; GFX8-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v7, v[4:5]
-; GFX8-NEXT: flat_load_dword v6, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 4, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: flat_load_dword v2, v[6:7]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[6:7], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_mov_b32_e32 v3, v1
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v2, v0
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB23_4
; GFX8-NEXT: ; %bb.5: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB23_2
; GFX8-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 4, v4
-; GFX8-NEXT: buffer_load_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc
+; GFX8-NEXT: v_add_u32_e32 v3, vcc, 4, v2
+; GFX8-NEXT: buffer_load_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
-; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; GFX8-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
-; GFX8-NEXT: buffer_store_dword v1, v5, s[0:3], 0 offen
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX8-NEXT: buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX8-NEXT: buffer_store_dword v1, v3, s[0:3], 0 offen
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -5241,29 +5214,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5272,26 +5245,25 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB24_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
@@ -5338,29 +5310,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB24_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -5368,78 +5340,76 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB24_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB24_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX10-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX10-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v1
; GFX10-NEXT: s_and_saveexec_b32 s4, vcc_lo
; GFX10-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX10-NEXT: s_cbranch_execz .LBB24_4
; GFX10-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX10-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX10-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v7, v5
-; GFX10-NEXT: v_mov_b32_e32 v6, v4
-; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX10-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX10-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v9, v3
+; GFX10-NEXT: v_mov_b32_e32 v8, v2
+; GFX10-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX10-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX10-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX10-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_2
; GFX10-NEXT: ; %bb.3: ; %Flow
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX10-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX10-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX10-NEXT: .LBB24_4: ; %Flow2
; GFX10-NEXT: s_andn2_saveexec_b32 s4, s4
; GFX10-NEXT: s_cbranch_execz .LBB24_6
; GFX10-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX10-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX10-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX10-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX10-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
+; GFX10-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX10-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX10-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
+; GFX10-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX10-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX10-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX10-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
; GFX10-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s4
-; GFX10-NEXT: v_mov_b32_e32 v0, v4
-; GFX10-NEXT: v_mov_b32_e32 v1, v5
+; GFX10-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v1, v3
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
@@ -5447,144 +5417,138 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: s_mov_b64 s[4:5], src_private_base
; GFX90A-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
-; GFX90A-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX90A-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
+; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB24_4
; GFX90A-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX90A-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX90A-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
; GFX90A-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX90A-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX90A-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_2
; GFX90A-NEXT: ; %bb.3: ; %Flow
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX90A-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX90A-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX90A-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX90A-NEXT: .LBB24_4: ; %Flow2
; GFX90A-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX90A-NEXT: s_cbranch_execz .LBB24_6
; GFX90A-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX90A-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; GFX90A-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
-; GFX90A-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_load_dword v5, v6, s[0:3], 0 offen offset:4
-; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX90A-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc
+; GFX90A-NEXT: buffer_load_dword v2, v4, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_load_dword v3, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX90A-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
-; GFX90A-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
-; GFX90A-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX90A-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX90A-NEXT: v_min_f64 v[0:1], v[0:1], v[6:7]
+; GFX90A-NEXT: buffer_store_dword v0, v4, s[0:3], 0 offen
+; GFX90A-NEXT: buffer_store_dword v1, v4, s[0:3], 0 offen offset:4
; GFX90A-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v4
-; GFX90A-NEXT: v_mov_b32_e32 v1, v5
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v3
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_6
-; GFX908-NEXT: .LBB24_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB24_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB24_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB24_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB24_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB24_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB24_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB24_2
-; GFX908-NEXT: .LBB24_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB24_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB24_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB24_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB24_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB24_6
@@ -5592,18 +5556,17 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -5612,37 +5575,37 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX7-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
+; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX7-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX7-NEXT: s_cbranch_execz .LBB24_4
; GFX7-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX7-NEXT: v_add_i32_e32 v4, vcc, 4, v0
-; GFX7-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX7-NEXT: flat_load_dword v5, v[4:5]
-; GFX7-NEXT: flat_load_dword v4, v[0:1]
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX7-NEXT: flat_load_dword v3, v[2:3]
+; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v7, v5
-; GFX7-NEXT: v_mov_b32_e32 v6, v4
-; GFX7-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX7-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX7-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX7-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB24_2
; GFX7-NEXT: ; %bb.3: ; %Flow
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX7-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX7-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX7-NEXT: .LBB24_4: ; %Flow2
; GFX7-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX7-NEXT: s_cbranch_execz .LBB24_6
@@ -5650,18 +5613,17 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX7-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX7-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX7-NEXT: v_add_i32_e32 v7, vcc, 4, v6
-; GFX7-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX7-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX7-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX7-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX7-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
+; GFX7-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX7-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX7-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX7-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX7-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v0, v4
-; GFX7-NEXT: v_mov_b32_e32 v1, v5
+; GFX7-NEXT: v_mov_b32_e32 v0, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, v3
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: s_setpc_b64 s[30:31]
%result = atomicrmw fmin ptr %ptr, double %val syncscope("agent") seq_cst, !amdgpu.no.remote.memory !0
@@ -5676,29 +5638,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5707,26 +5669,25 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB25_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_6
; GFX12-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5773,29 +5734,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX11-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX11-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB25_4
; GFX11-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX11-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX11-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX11-NEXT: s_mov_b32 s1, 0
; GFX11-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX11-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX11-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -5803,23 +5764,22 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX11-NEXT: ; %bb.3: ; %Flow
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX11-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX11-NEXT: .LBB25_4: ; %Flow2
; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-NEXT: s_cbranch_execz .LBB25_6
; GFX11-NEXT: ; %bb.5: ; %atomicrmw.private
; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
-; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX11-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX11-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
+; GFX11-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX11-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX11-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5909,95 +5869,90 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v5, v1
+; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], src_private_base
-; GFX908-NEXT: v_mov_b32_e32 v4, v0
-; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v5
-; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX908-NEXT: v_cmp_ne_u32_e32 vcc, s5, v1
+; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX908-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX908-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_3
-; GFX908-NEXT: ; %bb.1: ; %Flow2
-; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_6
-; GFX908-NEXT: .LBB25_2: ; %atomicrmw.phi
-; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX908-NEXT: s_setpc_b64 s[30:31]
-; GFX908-NEXT: .LBB25_3: ; %atomicrmw.global
-; GFX908-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX908-NEXT: s_cbranch_execz .LBB25_4
+; GFX908-NEXT: ; %bb.1: ; %atomicrmw.global
+; GFX908-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
-; GFX908-NEXT: .LBB25_4: ; %atomicrmw.start
+; GFX908-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v9, v1
-; GFX908-NEXT: v_mov_b32_e32 v8, v0
-; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
-; GFX908-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX908-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
-; GFX908-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
+; GFX908-NEXT: v_mov_b32_e32 v9, v3
+; GFX908-NEXT: v_mov_b32_e32 v8, v2
+; GFX908-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX908-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX908-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
-; GFX908-NEXT: s_cbranch_execnz .LBB25_4
-; GFX908-NEXT: ; %bb.5: ; %Flow
+; GFX908-NEXT: s_cbranch_execnz .LBB25_2
+; GFX908-NEXT: ; %bb.3: ; %Flow
; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX908-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GFX908-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX908-NEXT: .LBB25_4: ; %Flow2
; GFX908-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
-; GFX908-NEXT: s_cbranch_execz .LBB25_2
-; GFX908-NEXT: .LBB25_6: ; %atomicrmw.private
-; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
-; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc
-; GFX908-NEXT: buffer_load_dword v0, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_load_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: s_cbranch_execz .LBB25_6
+; GFX908-NEXT: ; %bb.5: ; %atomicrmw.private
+; GFX908-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
+; GFX908-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_load_dword v3, v6, s[0:3], 0 offen offset:4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[4:5], v[0:1], v[0:1]
-; GFX908-NEXT: v_min_f64 v[2:3], v[4:5], v[2:3]
-; GFX908-NEXT: buffer_store_dword v2, v6, s[0:3], 0 offen
-; GFX908-NEXT: buffer_store_dword v3, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX908-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
+; GFX908-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
+; GFX908-NEXT: buffer_store_dword v1, v6, s[0:3], 0 offen offset:4
+; GFX908-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX908-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v0, v2
+; GFX908-NEXT: v_mov_b32_e32 v1, v3
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0xc0
; GFX8-NEXT: s_load_dword s4, s[4:5], 0x0
-; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, s4, v1
; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB25_4
; GFX8-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 4, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v5, v[4:5]
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 4, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dword v3, v[2:3]
+; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
-; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_max_f64 v[2:3], v[8:9], v[8:9]
+; GFX8-NEXT: v_min_f64 v[6:7], v[2:3], v[4:5]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB25_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX8-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX8-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX8-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX8-NEXT: .LBB25_4: ; %Flow2
; GFX8-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; GFX8-NEXT: s_cbranch_execz .LBB25_6
@@ -6005,18 +5960,17 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX8-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX8-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc
; GFX8-NEXT: v_add_u32_e32 v7, vcc, 4, v6
-; GFX8-NEXT: buffer_load_dword v4, v6, s[0:3], 0 offen
-; GFX8-NEXT: buffer_load_dword v5, v7, s[0:3], 0 offen
-; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: buffer_load_dword v2, v6, s[0:3], 0 offen
+; GFX8-NEXT: buffer_load_dword v3, v7, s[0:3], 0 offen
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
-; GFX8-NEXT: v_min_f64 v[0:1], v[2:3], v[0:1]
+; GFX8-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX8-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX8-NEXT: buffer_store_dword v0, v6, s[0:3], 0 offen
; GFX8-NEXT: buffer_store_dword v1, v7, s[0:3], 0 offen
; GFX8-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v1, v5
+; GFX8-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -6079,8 +6033,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6092,12 +6047,11 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6125,8 +6079,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6138,12 +6093,11 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6175,13 +6129,13 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_min_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -6203,8 +6157,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6216,12 +6171,11 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6245,8 +6199,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6258,12 +6213,11 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6287,6 +6241,7 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -6298,10 +6253,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6330,13 +6284,13 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_min_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
@@ -6364,14 +6318,14 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -6398,17 +6352,17 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6467,33 +6421,35 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -6504,7 +6460,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6517,24 +6473,25 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6561,67 +6518,69 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -6632,18 +6591,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -6653,12 +6613,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6683,9 +6642,10 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -6694,10 +6654,9 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6717,36 +6676,36 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6762,14 +6721,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -6797,17 +6756,17 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6868,33 +6827,35 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -6905,7 +6866,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6918,24 +6879,25 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -6963,67 +6925,69 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -7034,18 +6998,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7055,12 +7020,11 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7085,9 +7049,10 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -7096,10 +7061,9 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7119,36 +7083,36 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7164,14 +7128,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -7199,17 +7163,17 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -7271,8 +7235,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7284,10 +7249,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7316,8 +7280,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7329,10 +7294,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7365,13 +7329,13 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_min_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -7392,8 +7356,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7405,10 +7370,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7433,8 +7397,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7446,10 +7411,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -7474,6 +7438,7 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -7485,9 +7450,8 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX10-NEXT: v_min_f16_e32 v3, v3, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7516,13 +7480,13 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_min_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
@@ -7549,13 +7513,13 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX908-NEXT: v_min_f16_e32 v3, v3, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
@@ -7582,16 +7546,16 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
+; GFX8-NEXT: v_min_f16_e32 v3, v3, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -7649,36 +7613,38 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7695,36 +7661,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7741,30 +7708,30 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB30_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7774,37 +7741,39 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7816,37 +7785,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7858,31 +7827,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB30_1
@@ -7893,31 +7862,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7927,31 +7896,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7961,32 +7930,32 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8040,36 +8009,38 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8086,36 +8057,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8133,30 +8105,30 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB31_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8166,37 +8138,39 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8208,37 +8182,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8250,31 +8224,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB31_1
@@ -8285,31 +8259,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8319,31 +8293,31 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8353,32 +8327,32 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX8-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8433,15 +8407,15 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8468,15 +8442,15 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8498,44 +8472,44 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB32_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8558,15 +8532,15 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8590,23 +8564,23 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX10-NEXT: v_min_f16_e32 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX10-NEXT: v_min_f16_e32 v0, v0, v1
; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX10-NEXT: v_and_or_b32 v5, 0xffff0000, v6, v0
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB32_1
@@ -8617,27 +8591,27 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -8645,14 +8619,14 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f16_e32 v3, v3, v2
; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -8673,19 +8647,19 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
+; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v5, v2, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
@@ -8734,14 +8708,14 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8767,24 +8741,24 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8797,24 +8771,24 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX942-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB33_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8825,14 +8799,14 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -8854,25 +8828,25 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8886,23 +8860,23 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB33_1
@@ -8913,22 +8887,22 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2046
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8938,22 +8912,22 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX908-LABEL: flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2046
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f16_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2046 glc
+; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX908-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8965,22 +8939,22 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9027,34 +9001,36 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -9065,7 +9041,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9078,24 +9054,25 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -9123,67 +9100,69 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB34_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -9194,18 +9173,19 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -9215,12 +9195,11 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -9245,9 +9224,10 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v5, v[0:1]
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -9256,10 +9236,9 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9279,39 +9258,39 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX90A-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9327,14 +9306,14 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
@@ -9362,17 +9341,17 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -9433,37 +9412,39 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9480,37 +9461,38 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9527,30 +9509,30 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB35_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9560,37 +9542,39 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9602,37 +9586,37 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9644,31 +9628,31 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB35_1
@@ -9679,34 +9663,34 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v4, v[0:1], v[4:5] glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9716,31 +9700,31 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX908-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9750,32 +9734,32 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX8-LABEL: flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13287,15 +13271,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13314,43 +13298,43 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX942-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB46_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13370,14 +13354,14 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13395,25 +13379,25 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -13421,13 +13405,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13445,21 +13429,21 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_max_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
@@ -13518,15 +13502,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13545,43 +13529,43 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX942-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB47_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -13602,21 +13586,21 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_min_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_min_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB47_1
@@ -13627,25 +13611,25 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13653,13 +13637,13 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13679,21 +13663,21 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
@@ -13754,15 +13738,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13785,25 +13769,25 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v2, v2
; GFX942-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v0, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v1, v7, v7
+; GFX942-NEXT: v_pk_max_f16 v0, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v6, v1, v0
+; GFX942-NEXT: v_pk_min_f16 v2, v0, v1
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB48_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13816,22 +13800,22 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_pk_max_f16 v1, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX11-NEXT: v_pk_min_f16 v0, v5, v0
+; GFX11-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX11-NEXT: v_pk_min_f16 v5, v0, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13845,21 +13829,21 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_min_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_min_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB48_1
@@ -13877,12 +13861,12 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_min_f16 v0, v3, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -13905,17 +13889,17 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v2, v2
; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
-; GFX908-NEXT: v_pk_min_f16 v0, v0, v5
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX908-NEXT: v_pk_min_f16 v5, v0, v1
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB48_1
@@ -13930,21 +13914,21 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB48_1
@@ -14004,21 +13988,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14031,23 +14015,23 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB49_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14057,22 +14041,22 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14084,21 +14068,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB49_1
@@ -14109,20 +14093,20 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14132,20 +14116,20 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14155,24 +14139,24 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14225,21 +14209,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14252,23 +14236,23 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14278,22 +14262,22 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14307,21 +14291,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -14332,20 +14316,20 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14355,20 +14339,20 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14380,24 +14364,24 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14453,21 +14437,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14484,25 +14468,25 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14515,22 +14499,22 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14544,21 +14528,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -14576,12 +14560,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v2
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v0, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX90A-NEXT: v_pk_min_f16 v0, v3, v0
+; GFX90A-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX90A-NEXT: v_pk_min_f16 v0, v0, v2
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -14604,12 +14588,12 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v0, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX908-NEXT: v_pk_min_f16 v0, v5, v0
+; GFX908-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX908-NEXT: v_pk_min_f16 v0, v0, v2
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14627,24 +14611,24 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14701,15 +14685,15 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14729,43 +14713,43 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX942-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -14786,21 +14770,21 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v1, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_pk_max_f16 v0, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v1, v1
-; GFX10-NEXT: v_pk_min_f16 v0, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_pk_max_f16 v0, v6, v6
+; GFX10-NEXT: v_pk_min_f16 v5, v0, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB52_1
@@ -14811,28 +14795,28 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14840,13 +14824,13 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14866,21 +14850,21 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -14940,22 +14924,22 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14968,23 +14952,23 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14994,22 +14978,22 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX11-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15023,21 +15007,21 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -15048,23 +15032,23 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15074,20 +15058,20 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15099,24 +15083,24 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15177,41 +15161,43 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15230,38 +15216,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15275,87 +15262,88 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX942-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15370,39 +15358,41 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15417,34 +15407,34 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -15456,41 +15446,41 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15498,33 +15488,33 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
@@ -15538,34 +15528,34 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
@@ -15629,41 +15619,43 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15682,38 +15674,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15727,87 +15720,88 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX942-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15822,39 +15816,41 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15870,35 +15866,35 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -15909,41 +15905,41 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX90A-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15951,33 +15947,33 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB55_1
@@ -15993,34 +15989,34 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
@@ -16086,41 +16082,43 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16139,38 +16137,39 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16188,41 +16187,41 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v7
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX942-NEXT: v_min_f32_e32 v0, v1, v0
-; GFX942-NEXT: v_min_f32_e32 v1, v6, v3
-; GFX942-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX942-NEXT: v_add3_u32 v3, v3, v0, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v1, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX942-NEXT: v_add3_u32 v7, v7, v0, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v2, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v6, v1, v0, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v2, v0, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16235,6 +16234,8 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -16242,36 +16243,36 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_and_b32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_min_f32 v0, v5, v0 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16287,6 +16288,8 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -16294,33 +16297,33 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_lshlrev_b32 v0, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v0, v6, v0 :: v_dual_and_b32 v7, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16335,35 +16338,35 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -16381,28 +16384,28 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -16425,33 +16428,33 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v0, s9
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -16466,34 +16469,34 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -16558,43 +16561,42 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16611,40 +16613,40 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16657,39 +16659,39 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX942-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16699,44 +16701,44 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16749,41 +16751,41 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16796,35 +16798,35 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX10-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -16835,36 +16837,36 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16874,36 +16876,36 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16913,37 +16915,37 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX8-LABEL: flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17000,43 +17002,42 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17053,40 +17054,40 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17099,39 +17100,39 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB58_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17141,44 +17142,44 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17191,41 +17192,41 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17240,35 +17241,35 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB58_1
@@ -17279,36 +17280,36 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17318,36 +17319,36 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17359,37 +17360,37 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17449,43 +17450,42 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17502,40 +17502,40 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17552,41 +17552,41 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB59_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17599,44 +17599,44 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17652,41 +17652,41 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17701,35 +17701,35 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB59_1
@@ -17747,28 +17747,28 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_min_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_min_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -17791,28 +17791,28 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_min_f32_e32 v0, v5, v0
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX908-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -17830,37 +17830,37 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17921,42 +17921,44 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17975,39 +17977,40 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18021,87 +18024,88 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX942-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB60_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18116,39 +18120,41 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18164,35 +18170,35 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_min_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB60_1
@@ -18203,44 +18209,44 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX90A-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18248,33 +18254,33 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB60_1
@@ -18290,34 +18296,34 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB60_1
@@ -18382,44 +18388,43 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18436,41 +18441,41 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18483,39 +18488,39 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX942-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB61_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18525,44 +18530,44 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18575,41 +18580,41 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18624,35 +18629,35 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB61_1
@@ -18663,39 +18668,39 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX90A-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18705,36 +18710,36 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX908-LABEL: flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18746,37 +18751,37 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index 50b070e63cc2c..2c64aaf801f1f 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -14580,41 +14580,43 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14633,38 +14635,39 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14678,87 +14681,88 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX942-LABEL: flat_agent_atomic_fsub_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14773,39 +14777,41 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -14820,34 +14826,34 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -14859,41 +14865,41 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX90A-LABEL: flat_agent_atomic_fsub_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fsub_ret_v2bf16:
@@ -14901,33 +14907,33 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
@@ -14941,34 +14947,34 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16(ptr %ptr, <2 x bfloat> %v
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
@@ -15032,41 +15038,43 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15085,38 +15093,39 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15130,87 +15139,88 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX942-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15225,39 +15235,41 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15273,35 +15285,35 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_sub_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -15312,41 +15324,41 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -15354,33 +15366,33 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB51_1
@@ -15396,34 +15408,34 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
@@ -15489,41 +15501,43 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15542,38 +15556,39 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15591,41 +15606,41 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v7, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v7
-; GFX942-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v7
-; GFX942-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX942-NEXT: v_sub_f32_e32 v1, v6, v3
-; GFX942-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX942-NEXT: v_add3_u32 v3, v3, v0, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v1, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; GFX942-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX942-NEXT: v_add3_u32 v7, v7, v0, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v2, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v0, v0
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v1, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v3, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v6, v1, v0, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v2, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v2, v0, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[6:7] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v0, v[4:5], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v7
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v7, v0
+; GFX942-NEXT: v_mov_b32_e32 v3, v0
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -15638,6 +15653,8 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -15645,36 +15662,36 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_and_b32 v0, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_sub_f32 v0, v5, v0 :: v_dual_lshlrev_b32 v7, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15690,6 +15707,8 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: flat_load_b32 v0, v[3:4]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
@@ -15697,33 +15716,33 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX11-FAKE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_lshlrev_b32 v0, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v0, v6, v0 :: v_dual_and_b32 v7, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[0:1] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15738,35 +15757,35 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_sub_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB52_1
@@ -15784,28 +15803,28 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -15828,33 +15847,33 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v0, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v1, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v1, v0
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v0, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
-; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
-; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX908-NEXT: v_mov_b32_e32 v6, v0
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v0, s9
+; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
+; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB52_1
@@ -15869,34 +15888,34 @@ define <2 x bfloat> @flat_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr %ptr,
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -15961,43 +15980,42 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16014,40 +16032,40 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16060,39 +16078,39 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX942-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1]
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16102,44 +16120,44 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16152,41 +16170,41 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16199,35 +16217,35 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX10-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -16238,36 +16256,36 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX90A-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1]
+; GFX90A-NEXT: flat_load_dword v3, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16277,36 +16295,36 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX908-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1]
+; GFX908-NEXT: flat_load_dword v3, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16316,37 +16334,37 @@ define void @flat_agent_atomic_fsub_noret_v2bf16(ptr %ptr, <2 x bfloat> %val) #0
; GFX8-LABEL: flat_agent_atomic_fsub_noret_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16403,43 +16421,42 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16456,40 +16473,40 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16502,39 +16519,39 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX942-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16544,44 +16561,44 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16594,41 +16611,41 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16643,35 +16660,35 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -16682,36 +16699,36 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX90A-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16721,36 +16738,36 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX908-LABEL: flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16762,37 +16779,37 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x b
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16852,43 +16869,42 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16905,40 +16921,40 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16955,41 +16971,41 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX942-NEXT: s_movk_i32 s0, 0xf800
; GFX942-NEXT: s_nop 0
; GFX942-NEXT: v_addc_co_u32_e32 v5, vcc, -1, v1, vcc
-; GFX942-NEXT: flat_load_dword v5, v[4:5]
+; GFX942-NEXT: flat_load_dword v3, v[4:5]
; GFX942-NEXT: s_mov_b32 s1, -1
; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, s[0:1]
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] sc0
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17002,44 +17018,44 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17055,41 +17071,41 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX11-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17104,35 +17120,35 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -17150,28 +17166,28 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX90A-NEXT: v_mov_b32_e32 v0, v4
; GFX90A-NEXT: flat_load_dword v1, v[0:1]
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX90A-NEXT: v_sub_f32_e32 v0, v3, v0
-; GFX90A-NEXT: v_sub_f32_e32 v3, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX90A-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX90A-NEXT: v_sub_f32_e32 v0, v0, v3
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX90A-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX90A-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v0, v3, v0, s9
+; GFX90A-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX90A-NEXT: flat_atomic_cmpswap v0, v[4:5], v[0:1] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
@@ -17194,28 +17210,28 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX908-NEXT: v_mov_b32_e32 v0, v3
; GFX908-NEXT: flat_load_dword v1, v[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX908-NEXT: v_sub_f32_e32 v0, v5, v0
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v0, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX908-NEXT: v_sub_f32_e32 v0, v0, v5
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v2
+; GFX908-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v0, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
-; GFX908-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v0, v5, v0, s9
+; GFX908-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v0, v6, v0, s9
; GFX908-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -17233,37 +17249,37 @@ define void @flat_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr %ptr, <2 x b
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17324,42 +17340,44 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17378,39 +17396,40 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17424,87 +17443,88 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX942-LABEL: flat_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17519,39 +17539,41 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17567,35 +17589,35 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: flat_load_dword v0, v[3:4]
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX10-NEXT: v_sub_f32_e32 v0, v6, v0
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v0, v5, v0, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v0, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX10-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -17606,44 +17628,44 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX90A-LABEL: flat_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: flat_system_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -17651,33 +17673,33 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -17693,34 +17715,34 @@ define <2 x bfloat> @flat_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr %ptr,
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -17785,44 +17807,43 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17839,41 +17860,41 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17886,39 +17907,39 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX942-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX942-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 sc0 sc1
+; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17928,44 +17949,44 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17978,41 +17999,41 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 glc
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18027,35 +18048,35 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fc, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
-; GFX10-NEXT: flat_load_dword v4, v[0:1]
+; GFX10-NEXT: flat_load_dword v3, v[0:1]
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX10-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX10-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -18066,39 +18087,39 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX90A-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: flat_load_dword v5, v[0:1] offset:2044
+; GFX90A-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: flat_atomic_cmpswap v3, v[0:1], v[4:5] offset:2044 glc
+; GFX90A-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18108,36 +18129,36 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX908-LABEL: flat_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: flat_load_dword v4, v[0:1] offset:2044
+; GFX908-NEXT: flat_load_dword v3, v[0:1] offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18149,37 +18170,37 @@ define void @flat_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr %ptr, <2 x
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
index de76d98923b48..78aefe033755a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-atomics.ll
@@ -13390,25 +13390,23 @@ define float @flat_atomic_fmax_f32_saddr_rtn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
; GFX950-NEXT: .LBB118_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_max_f32_e32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB118_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmax ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -13430,20 +13428,19 @@ define void @flat_atomic_fmax_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
; GFX950-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_max_f32_e32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB119_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13469,25 +13466,23 @@ define float @flat_atomic_fmin_f32_saddr_rtn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
; GFX950-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_min_f32_e32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB120_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x float], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmin ptr %gep.0, float %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -13509,20 +13504,19 @@ define void @flat_atomic_fmin_f32_saddr_nortn(ptr inreg %ptr, float %data) {
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_max_f32_e32 v4, v0, v0
; GFX950-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_max_f32_e32 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_max_f32_e32 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_min_f32_e32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB121_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13602,55 +13596,54 @@ define <2 x half> @flat_atomic_fmax_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
; GFX1250-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
-; GFX1250-NEXT: v_pk_max_num_f16 v1, v0, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v3, v5, v5
-; GFX1250-NEXT: v_pk_max_num_f16 v4, v3, v1
+; GFX1250-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-NEXT: v_pk_max_num_f16 v4, v0, v1
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX1250-NEXT: s_cbranch_execnz .LBB124_1
; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmax_v2f16_saddr_rtn:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_max_f16 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB124_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmax ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -13662,22 +13655,22 @@ define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v0
; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
+; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
; GFX1250-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_max_num_f16 v4, v3, v3
+; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v2, v4, v2
+; GFX1250-NEXT: v_pk_max_num_f16 v0, v0, v3
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
@@ -13690,21 +13683,21 @@ define void @flat_atomic_fmax_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_max_f16 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_max_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB125_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13720,55 +13713,54 @@ define <2 x half> @flat_atomic_fmin_v2f16_saddr_rtn(ptr inreg %ptr, <2 x half> %
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, 0
; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX1250-NEXT: flat_load_b32 v0, v2, s[0:1] offset:40
; GFX1250-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
-; GFX1250-NEXT: v_pk_max_num_f16 v1, v0, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_max_num_f16 v3, v5, v5
-; GFX1250-NEXT: v_pk_min_num_f16 v4, v3, v1
+; GFX1250-NEXT: v_mov_b32_e32 v5, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_max_num_f16 v0, v5, v5
+; GFX1250-NEXT: v_pk_min_num_f16 v4, v0, v1
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v1, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[4:5], s[0:1] offset:40 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v5
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v5
; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
; GFX1250-NEXT: s_cbranch_execnz .LBB126_1
; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s2
-; GFX1250-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX950-LABEL: flat_atomic_fmin_v2f16_saddr_rtn:
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_min_f16 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB126_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x half>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmin ptr %gep.0, <2 x half> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -13780,22 +13772,22 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v2, 0
+; GFX1250-NEXT: v_pk_max_num_f16 v3, v0, v0
; GFX1250-NEXT: s_mov_b32 s2, 0
-; GFX1250-NEXT: flat_load_b32 v3, v1, s[0:1] offset:40
+; GFX1250-NEXT: flat_load_b32 v1, v2, s[0:1] offset:40
; GFX1250-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-NEXT: v_pk_max_num_f16 v2, v0, v0
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_pk_max_num_f16 v4, v3, v3
+; GFX1250-NEXT: v_pk_max_num_f16 v0, v1, v1
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_min_num_f16 v2, v4, v2
+; GFX1250-NEXT: v_pk_min_num_f16 v0, v0, v3
; GFX1250-NEXT: s_wait_storecnt 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: flat_atomic_cmpswap_b32 v2, v1, v[2:3], s[0:1] offset:40 th:TH_ATOMIC_RETURN
+; GFX1250-NEXT: flat_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:40 th:TH_ATOMIC_RETURN
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-NEXT: v_mov_b32_e32 v3, v2
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
+; GFX1250-NEXT: v_mov_b32_e32 v1, v0
; GFX1250-NEXT: s_or_b32 s2, vcc_lo, s2
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
@@ -13808,21 +13800,21 @@ define void @flat_atomic_fmin_v2f16_saddr_nortn(ptr inreg %ptr, <2 x half> %data
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_pk_max_f16 v4, v0, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_pk_max_f16 v1, v0, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_pk_max_f16 v2, v3, v3
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_pk_min_f16 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_pk_max_f16 v0, v1, v1
+; GFX950-NEXT: s_nop 0
+; GFX950-NEXT: v_pk_min_f16 v0, v0, v4
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB127_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13929,30 +13921,28 @@ define <2 x bfloat> @flat_atomic_fmax_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB130_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX950-NEXT: v_max_f32_e32 v2, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB130_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmax ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -13989,25 +13979,24 @@ define void @flat_atomic_fmax_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %d
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB131_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_max_f32_e32 v1, v2, v1
-; GFX950-NEXT: v_max_f32_e32 v2, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB131_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14050,30 +14039,28 @@ define <2 x bfloat> @flat_atomic_fmin_v2bf16_saddr_rtn(ptr inreg %ptr, <2 x bflo
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB132_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX950-NEXT: v_min_f32_e32 v2, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB132_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX950-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v0, v1
; GFX950-NEXT: s_setpc_b64 s[30:31]
%gep.0 = getelementptr inbounds [512 x <2 x bfloat>], ptr %ptr, i64 0, i64 10
%result = atomicrmw fmin ptr %gep.0, <2 x bfloat> %data syncscope("workgroup") seq_cst, align 8, !amdgpu.no.fine.grained.memory !0
@@ -14110,25 +14097,24 @@ define void @flat_atomic_fmin_v2bf16_saddr_nortn(ptr inreg %ptr, <2 x bfloat> %d
; GFX950: ; %bb.0:
; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX950-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX950-NEXT: flat_load_dword v3, v[2:3] offset:40
+; GFX950-NEXT: flat_load_dword v1, v[2:3] offset:40
; GFX950-NEXT: s_mov_b64 s[2:3], 0
+; GFX950-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX950-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX950-NEXT: .p2align 5, , 4
; GFX950-NEXT: .LBB133_1: ; %atomicrmw.start
; GFX950-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX950-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX950-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX950-NEXT: v_min_f32_e32 v1, v2, v1
-; GFX950-NEXT: v_min_f32_e32 v2, v7, v6
-; GFX950-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-NEXT: v_cvt_pk_bf16_f32 v2, v2, v1
-; GFX950-NEXT: flat_atomic_cmpswap v1, v[4:5], v[2:3] offset:40 sc0
+; GFX950-NEXT: v_and_b32_e32 v0, 0xffff0000, v1
+; GFX950-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX950-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX950-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX950-NEXT: v_cvt_pk_bf16_f32 v0, v6, v0
+; GFX950-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:40 sc0
; GFX950-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX950-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
; GFX950-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX950-NEXT: v_mov_b32_e32 v3, v1
+; GFX950-NEXT: v_mov_b32_e32 v1, v0
; GFX950-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX950-NEXT: s_cbranch_execnz .LBB133_1
; GFX950-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
index 847262c5696d9..ff055e01ed793 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
@@ -1345,20 +1345,18 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB34_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_subrev_i32_e32 v2, vcc, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB34_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1370,20 +1368,18 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB34_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_subrev_u32_e32 v2, vcc, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB34_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1395,20 +1391,18 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB34_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_subrev_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB34_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1426,24 +1420,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB35_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_subrev_i32_e32 v2, vcc, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB35_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
@@ -1453,24 +1445,22 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB35_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_subrev_u32_e32 v2, vcc, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB35_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_noret_offset_scalar:
@@ -1478,20 +1468,18 @@ define amdgpu_gfx void @flat_atomic_sub_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB35_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_subrev_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB35_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1510,17 +1498,17 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB36_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB36_1
@@ -1535,17 +1523,17 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB36_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_subrev_u32_e32 v3, vcc, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB36_1
@@ -1560,17 +1548,17 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB36_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_subrev_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB36_1
@@ -1587,26 +1575,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB37_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_subrev_i32_e32 v3, vcc, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB37_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
@@ -1614,26 +1600,24 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB37_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_subrev_u32_e32 v3, vcc, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB37_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_sub_i32_ret_offset_scalar:
@@ -1643,17 +1627,17 @@ define amdgpu_gfx i32 @flat_atomic_sub_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB37_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_subrev_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB37_1
@@ -2023,20 +2007,18 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB44_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2048,20 +2030,18 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB44_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2073,20 +2053,18 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB44_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB44_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2104,24 +2082,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB45_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_noret_offset_scalar:
@@ -2131,24 +2107,22 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB45_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_noret_offset_scalar:
@@ -2156,20 +2130,18 @@ define amdgpu_gfx void @flat_atomic_and_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB45_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB45_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2188,17 +2160,17 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB46_1
@@ -2213,17 +2185,17 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB46_1
@@ -2238,17 +2210,17 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB46_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB46_1
@@ -2265,26 +2237,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB47_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_and_i32_ret_offset_scalar:
@@ -2292,26 +2262,24 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB47_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_and_i32_ret_offset_scalar:
@@ -2321,17 +2289,17 @@ define amdgpu_gfx i32 @flat_atomic_and_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB47_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB47_1
@@ -2713,21 +2681,19 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB54_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_not_b32_e32 v0, v0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: v_not_b32_e32 v2, v2
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB54_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2739,21 +2705,19 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB54_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_not_b32_e32 v0, v0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: v_not_b32_e32 v2, v2
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB54_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2765,21 +2729,19 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB54_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: v_not_b32_e32 v2, v2
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB54_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2797,25 +2759,23 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_not_b32_e32 v0, v0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN1-NEXT: v_not_b32_e32 v2, v2
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB55_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
@@ -2825,25 +2785,23 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_not_b32_e32 v0, v0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN2-NEXT: v_not_b32_e32 v2, v2
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB55_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_noret_offset_scalar:
@@ -2851,21 +2809,19 @@ define amdgpu_gfx void @flat_atomic_nand_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB55_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_and_b32_e32 v2, s6, v3
+; GCN3-NEXT: v_not_b32_e32 v2, v2
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB55_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2884,18 +2840,18 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB56_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_not_b32_e32 v0, v0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN1-NEXT: v_not_b32_e32 v3, v0
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB56_1
@@ -2910,18 +2866,18 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB56_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_not_b32_e32 v0, v0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN2-NEXT: v_not_b32_e32 v3, v0
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB56_1
@@ -2936,18 +2892,18 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB56_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN3-NEXT: v_not_b32_e32 v3, v0
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB56_1
@@ -2964,27 +2920,25 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB57_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_not_b32_e32 v0, v0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN1-NEXT: v_not_b32_e32 v3, v0
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB57_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
@@ -2992,27 +2946,25 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB57_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_not_b32_e32 v0, v0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN2-NEXT: v_not_b32_e32 v3, v0
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB57_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_nand_i32_ret_offset_scalar:
@@ -3022,18 +2974,18 @@ define amdgpu_gfx i32 @flat_atomic_nand_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB57_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_not_b32_e32 v0, v0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s6, v4
+; GCN3-NEXT: v_not_b32_e32 v3, v0
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB57_1
@@ -3488,20 +3440,18 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB64_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB64_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3513,20 +3463,18 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB64_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB64_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3538,20 +3486,18 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_scalar(ptr inreg %ptr, i32 inre
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB64_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB64_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3569,24 +3515,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB65_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB65_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_noret_offset_scalar:
@@ -3596,24 +3540,22 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB65_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB65_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_noret_offset_scalar:
@@ -3621,20 +3563,18 @@ define amdgpu_gfx void @flat_atomic_or_i32_noret_offset_scalar(ptr inreg %out, i
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB65_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_or_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB65_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3653,17 +3593,17 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB66_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB66_1
@@ -3678,17 +3618,17 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB66_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB66_1
@@ -3703,17 +3643,17 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_scalar(ptr inreg %ptr, i32 inreg %
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB66_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB66_1
@@ -3730,26 +3670,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB67_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB67_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_or_i32_ret_offset_scalar:
@@ -3757,26 +3695,24 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB67_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB67_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_or_i32_ret_offset_scalar:
@@ -3786,17 +3722,17 @@ define amdgpu_gfx i32 @flat_atomic_or_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB67_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_or_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB67_1
@@ -4166,20 +4102,18 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB74_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4191,20 +4125,18 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB74_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4216,20 +4148,18 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB74_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB74_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4247,24 +4177,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB75_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
@@ -4274,24 +4202,22 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB75_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_noret_offset_scalar:
@@ -4299,20 +4225,18 @@ define amdgpu_gfx void @flat_atomic_xor_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB75_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_xor_b32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB75_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4331,17 +4255,17 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB76_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB76_1
@@ -4356,17 +4280,17 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB76_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB76_1
@@ -4381,17 +4305,17 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB76_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB76_1
@@ -4408,26 +4332,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB77_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB77_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
@@ -4435,26 +4357,24 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB77_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB77_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_xor_i32_ret_offset_scalar:
@@ -4464,17 +4384,17 @@ define amdgpu_gfx i32 @flat_atomic_xor_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB77_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_xor_b32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB77_1
@@ -4844,20 +4764,18 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB84_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4869,20 +4787,18 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB84_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4894,20 +4810,18 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB84_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB84_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4925,24 +4839,22 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB85_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_noret_offset_scalar:
@@ -4952,24 +4864,22 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB85_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_noret_offset_scalar:
@@ -4977,20 +4887,18 @@ define amdgpu_gfx void @flat_atomic_max_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB85_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB85_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5009,17 +4917,17 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB86_1
@@ -5034,17 +4942,17 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB86_1
@@ -5059,17 +4967,17 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB86_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB86_1
@@ -5086,26 +4994,24 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB87_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_max_i32_ret_offset_scalar:
@@ -5113,26 +5019,24 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB87_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_max_i32_ret_offset_scalar:
@@ -5142,17 +5046,17 @@ define amdgpu_gfx i32 @flat_atomic_max_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB87_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_i32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB87_1
@@ -5178,21 +5082,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB88_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -5210,21 +5112,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB88_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -5240,21 +5140,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB88_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB88_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -5280,27 +5178,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB89_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -5317,27 +5213,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB89_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -5352,27 +5246,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB89_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB89_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -5394,21 +5286,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB90_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -5424,21 +5314,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB90_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -5454,21 +5342,19 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i32 %index)
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB90_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_max_i32_e32 v2, s2, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB90_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -5491,27 +5377,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB91_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_ret_addr64:
@@ -5526,27 +5410,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB91_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_ret_addr64:
@@ -5561,27 +5443,25 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB91_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB91_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -5948,20 +5828,18 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB98_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5973,20 +5851,18 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB98_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5998,20 +5874,18 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB98_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB98_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6029,24 +5903,22 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB99_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
@@ -6056,24 +5928,22 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB99_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_noret_offset_scalar:
@@ -6081,20 +5951,18 @@ define amdgpu_gfx void @flat_atomic_umax_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB99_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB99_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6113,17 +5981,17 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB100_1
@@ -6138,17 +6006,17 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB100_1
@@ -6163,17 +6031,17 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB100_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB100_1
@@ -6190,26 +6058,24 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB101_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
@@ -6217,26 +6083,24 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB101_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umax_i32_ret_offset_scalar:
@@ -6246,17 +6110,17 @@ define amdgpu_gfx i32 @flat_atomic_umax_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB101_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_max_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB101_1
@@ -6282,21 +6146,19 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_max_u32_e32 v2, s2, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB102_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -6314,21 +6176,19 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_max_u32_e32 v2, s2, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB102_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -6344,21 +6204,19 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i32
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB102_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_max_u32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_max_u32_e32 v2, s2, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB102_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -6384,27 +6242,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB103_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -6421,27 +6277,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB103_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -6456,27 +6310,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr %out, ptr %out2
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB103_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB103_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -6499,27 +6351,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB104_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_ret_addr64:
@@ -6534,27 +6384,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB104_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_ret_addr64:
@@ -6569,27 +6417,25 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr %out, ptr %out2, i32 %
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB104_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_max_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_max_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB104_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -6956,20 +6802,18 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB111_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6981,20 +6825,18 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB111_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7006,20 +6848,18 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_scalar(ptr inreg %ptr, i32 in
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB111_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB111_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7037,24 +6877,22 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB112_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
@@ -7064,24 +6902,22 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB112_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_noret_offset_scalar:
@@ -7089,20 +6925,18 @@ define amdgpu_gfx void @flat_atomic_umin_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB112_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_min_u32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB112_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7121,17 +6955,17 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB113_1
@@ -7146,17 +6980,17 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB113_1
@@ -7171,17 +7005,17 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB113_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB113_1
@@ -7198,26 +7032,24 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB114_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
@@ -7225,26 +7057,24 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB114_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_umin_i32_ret_offset_scalar:
@@ -7254,17 +7084,17 @@ define amdgpu_gfx i32 @flat_atomic_umin_i32_ret_offset_scalar(ptr inreg %out, i3
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB114_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_u32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_u32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB114_1
@@ -7634,20 +7464,18 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB121_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7659,20 +7487,18 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB121_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7684,20 +7510,18 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_scalar(ptr inreg %ptr, i32 inr
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB121_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB121_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7715,24 +7539,22 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB122_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_noret_offset_scalar:
@@ -7742,24 +7564,22 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB122_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_noret_offset_scalar:
@@ -7767,20 +7587,18 @@ define amdgpu_gfx void @flat_atomic_min_i32_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB122_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB122_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7799,17 +7617,17 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB123_1
@@ -7824,17 +7642,17 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB123_1
@@ -7849,17 +7667,17 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_scalar(ptr inreg %ptr, i32 inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB123_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB123_1
@@ -7876,26 +7694,24 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB124_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_min_i32_ret_offset_scalar:
@@ -7903,26 +7719,24 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB124_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_min_i32_ret_offset_scalar:
@@ -7932,17 +7746,17 @@ define amdgpu_gfx i32 @flat_atomic_min_i32_ret_offset_scalar(ptr inreg %out, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB124_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_min_i32_e32 v3, s6, v4
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB124_1
@@ -7968,21 +7782,19 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB125_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
@@ -8000,21 +7812,19 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB125_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
@@ -8030,21 +7840,19 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i32 %
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB125_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s2, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB125_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -8070,27 +7878,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB126_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -8107,27 +7913,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB126_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -8142,27 +7946,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1] offset:16
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB126_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB126_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -8175,78 +7977,72 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x9
+; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s6
+; GCN1-NEXT: v_mov_b32_e32 v1, s7
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_min_i32_e32 v0, s4, v1
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB127_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
-; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s6
+; GCN2-NEXT: v_mov_b32_e32 v1, s7
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_min_i32_e32 v0, s4, v1
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB127_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_min_i32_e32 v2, s2, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB127_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
@@ -8268,27 +8064,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: flat_load_dword v2, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[0:1], 0
; GCN1-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s0
-; GCN1-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN1-NEXT: s_cbranch_execnz .LBB128_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN1-NEXT: v_mov_b32_e32 v1, s2
-; GCN1-NEXT: v_mov_b32_e32 v2, s3
-; GCN1-NEXT: flat_store_dword v[1:2], v0
+; GCN1-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_ret_addr64:
@@ -8303,27 +8097,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: flat_load_dword v2, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[0:1], 0
; GCN2-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s0
-; GCN2-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN2-NEXT: s_cbranch_execnz .LBB128_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN2-NEXT: v_mov_b32_e32 v1, s2
-; GCN2-NEXT: v_mov_b32_e32 v2, s3
-; GCN2-NEXT: flat_store_dword v[1:2], v0
+; GCN2-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_ret_addr64:
@@ -8338,27 +8130,25 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr %out, ptr %out2, i32 %i
; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dword v0, v[0:1]
-; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: flat_load_dword v2, v[0:1]
+; GCN3-NEXT: s_mov_b64 s[0:1], 0
; GCN3-NEXT: .LBB128_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s0
-; GCN3-NEXT: v_min_i32_e32 v0, s6, v1
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
+; GCN3-NEXT: v_min_i32_e32 v2, s6, v3
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GCN3-NEXT: s_cbranch_execnz .LBB128_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN3-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN3-NEXT: v_mov_b32_e32 v1, s2
-; GCN3-NEXT: v_mov_b32_e32 v2, s3
-; GCN3-NEXT: flat_store_dword v[1:2], v0
+; GCN3-NEXT: s_or_b64 exec, exec, s[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v0, s2
+; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i32, ptr %out, i32 %index
@@ -8749,22 +8539,20 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s5
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB135_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v1
-; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_add_i32_e32 v2, vcc, 1, v3
+; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB135_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8776,22 +8564,20 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s5
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB135_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_add_u32_e32 v2, vcc, 1, v3
+; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB135_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8803,22 +8589,20 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1]
+; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB135_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_add_u32_e32 v0, 1, v1
-; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_add_u32_e32 v2, 1, v3
+; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB135_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8836,26 +8620,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
; GCN1-NEXT: s_addc_u32 s35, s5, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s34
; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB136_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v1
-; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_add_i32_e32 v2, vcc, 1, v3
+; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN1-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB136_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
@@ -8865,26 +8647,24 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
; GCN2-NEXT: s_addc_u32 s35, s5, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s34
; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB136_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_add_u32_e32 v2, vcc, 1, v3
+; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN2-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB136_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_noret_offset_scalar:
@@ -8892,22 +8672,20 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i32_noret_offset_scalar(ptr inreg
; GCN3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s4
; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
+; GCN3-NEXT: flat_load_dword v3, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB136_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_add_u32_e32 v0, 1, v1
-; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_add_u32_e32 v2, 1, v3
+; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; GCN3-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
+; GCN3-NEXT: v_mov_b32_e32 v3, v2
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB136_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8926,19 +8704,19 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB137_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v1
-; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB137_1
@@ -8953,19 +8731,19 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB137_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v4
+; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB137_1
@@ -8980,19 +8758,19 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB137_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_add_u32_e32 v0, 1, v1
-; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_add_u32_e32 v0, 1, v4
+; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB137_1
@@ -9009,28 +8787,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s34, s4, 16
; GCN1-NEXT: s_addc_u32 s35, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s34
-; GCN1-NEXT: v_mov_b32_e32 v1, s35
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[34:35], 0
; GCN1-NEXT: .LBB138_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v1
-; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, s34
-; GCN1-NEXT: v_mov_b32_e32 v3, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_mov_b32_e32 v4, v0
+; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v4
+; GCN1-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN1-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB138_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN1-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
@@ -9038,28 +8814,26 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s34, s4, 16
; GCN2-NEXT: s_addc_u32 s35, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s34
-; GCN2-NEXT: v_mov_b32_e32 v1, s35
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[34:35], 0
; GCN2-NEXT: .LBB138_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, s34
-; GCN2-NEXT: v_mov_b32_e32 v3, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_mov_b32_e32 v4, v0
+; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v4
+; GCN2-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN2-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB138_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
+; GCN2-NEXT: s_or_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_uinc_wrap_i32_ret_offset_scalar:
@@ -9069,19 +8843,19 @@ define amdgpu_gfx i32 @flat_atomic_uinc_wrap_i32_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB138_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_add_u32_e32 v0, 1, v1
-; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_mov_b32_e32 v4, v0
+; GCN3-NEXT: v_add_u32_e32 v0, 1, v4
+; GCN3-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; GCN3-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB138_1
@@ -9489,15 +9263,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s6
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: .LBB145_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v1
; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -9518,15 +9292,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s6
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: .LBB145_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v1
; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -9547,15 +9321,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v1, v[0:1]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB145_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v1
; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v4, s6
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -9576,63 +9350,59 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg
; GCN1-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT: s_add_u32 s36, s4, 16
-; GCN1-NEXT: s_addc_u32 s37, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s36
-; GCN1-NEXT: v_mov_b32_e32 v1, s37
-; GCN1-NEXT: flat_load_dword v1, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[38:39], 0
+; GCN1-NEXT: s_add_u32 s34, s4, 16
+; GCN1-NEXT: s_addc_u32 s35, s5, 0
+; GCN1-NEXT: v_mov_b32_e32 v0, s34
+; GCN1-NEXT: v_mov_b32_e32 v1, s35
+; GCN1-NEXT: flat_load_dword v3, v[0:1]
+; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s6
; GCN1-NEXT: .LBB146_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v1
-; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v4, s6
+; GCN1-NEXT: v_subrev_i32_e32 v2, vcc, 1, v3
+; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v3
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v2, s36
-; GCN1-NEXT: v_mov_b32_e32 v3, s37
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: v_mov_b32_e32 v3, v2
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB146_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[38:39]
+; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT: s_add_u32 s36, s4, 16
-; GCN2-NEXT: s_addc_u32 s37, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s36
-; GCN2-NEXT: v_mov_b32_e32 v1, s37
-; GCN2-NEXT: flat_load_dword v1, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[38:39], 0
+; GCN2-NEXT: s_add_u32 s34, s4, 16
+; GCN2-NEXT: s_addc_u32 s35, s5, 0
+; GCN2-NEXT: v_mov_b32_e32 v0, s34
+; GCN2-NEXT: v_mov_b32_e32 v1, s35
+; GCN2-NEXT: flat_load_dword v3, v[0:1]
+; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s6
; GCN2-NEXT: .LBB146_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v1
-; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v4, s6
+; GCN2-NEXT: v_subrev_u32_e32 v2, vcc, 1, v3
+; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v3
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v2, s36
-; GCN2-NEXT: v_mov_b32_e32 v3, s37
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: v_mov_b32_e32 v3, v2
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB146_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[38:39]
+; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_noret_offset_scalar:
@@ -9642,15 +9412,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i32_noret_offset_scalar(ptr inreg
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v1, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: .LBB146_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v1
; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v4, s6
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -9676,21 +9446,21 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN1-NEXT: v_mov_b32_e32 v1, s5
; GCN1-NEXT: flat_load_dword v0, v[0:1]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v3, s6
+; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s5
; GCN1-NEXT: .LBB147_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_subrev_i32_e32 v4, vcc, 1, v1
-; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v0, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, v0
+; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v5
+; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB147_1
@@ -9705,21 +9475,21 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN2-NEXT: v_mov_b32_e32 v1, s5
; GCN2-NEXT: flat_load_dword v0, v[0:1]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v3, s6
+; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s5
; GCN2-NEXT: .LBB147_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_subrev_u32_e32 v4, vcc, 1, v1
-; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v0, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, v0
+; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v5
+; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB147_1
@@ -9734,21 +9504,21 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_scalar(ptr inreg %ptr, i32
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v3, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB147_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v1
-; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, v0
+; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v5
+; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB147_1
@@ -9763,63 +9533,59 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %ou
; GCN1-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
; GCN1: ; %bb.0:
; GCN1-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN1-NEXT: s_add_u32 s36, s4, 16
-; GCN1-NEXT: s_addc_u32 s37, s5, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s36
-; GCN1-NEXT: v_mov_b32_e32 v1, s37
-; GCN1-NEXT: flat_load_dword v0, v[0:1]
-; GCN1-NEXT: s_mov_b64 s[38:39], 0
+; GCN1-NEXT: s_add_u32 s34, s4, 16
+; GCN1-NEXT: s_addc_u32 s35, s5, 0
+; GCN1-NEXT: v_mov_b32_e32 v1, s34
+; GCN1-NEXT: v_mov_b32_e32 v2, s35
+; GCN1-NEXT: flat_load_dword v0, v[1:2]
+; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v3, s6
; GCN1-NEXT: .LBB148_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v1, v0
-; GCN1-NEXT: v_subrev_i32_e32 v4, vcc, 1, v1
-; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN1-NEXT: v_mov_b32_e32 v0, s6
+; GCN1-NEXT: v_mov_b32_e32 v5, v0
+; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v5
+; GCN1-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v2, s36
-; GCN1-NEXT: v_mov_b32_e32 v3, s37
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN1-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN1-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB148_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN1-NEXT: s_or_b64 exec, exec, s[38:39]
+; GCN1-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_setpc_b64 s[30:31]
;
; GCN2-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
; GCN2: ; %bb.0:
; GCN2-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN2-NEXT: s_add_u32 s36, s4, 16
-; GCN2-NEXT: s_addc_u32 s37, s5, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s36
-; GCN2-NEXT: v_mov_b32_e32 v1, s37
-; GCN2-NEXT: flat_load_dword v0, v[0:1]
-; GCN2-NEXT: s_mov_b64 s[38:39], 0
+; GCN2-NEXT: s_add_u32 s34, s4, 16
+; GCN2-NEXT: s_addc_u32 s35, s5, 0
+; GCN2-NEXT: v_mov_b32_e32 v1, s34
+; GCN2-NEXT: v_mov_b32_e32 v2, s35
+; GCN2-NEXT: flat_load_dword v0, v[1:2]
+; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v3, s6
; GCN2-NEXT: .LBB148_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v1, v0
-; GCN2-NEXT: v_subrev_u32_e32 v4, vcc, 1, v1
-; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN2-NEXT: v_mov_b32_e32 v0, s6
+; GCN2-NEXT: v_mov_b32_e32 v5, v0
+; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v5
+; GCN2-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v2, s36
-; GCN2-NEXT: v_mov_b32_e32 v3, s37
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; GCN2-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN2-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB148_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
-; GCN2-NEXT: s_or_b64 exec, exec, s[38:39]
+; GCN2-NEXT: s_or_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_setpc_b64 s[30:31]
;
; GCN3-LABEL: flat_atomic_udec_wrap_i32_ret_offset_scalar:
@@ -9829,21 +9595,21 @@ define amdgpu_gfx i32 @flat_atomic_udec_wrap_i32_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: v_mov_b32_e32 v1, s5
; GCN3-NEXT: flat_load_dword v0, v[0:1] offset:16
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v3, s6
+; GCN3-NEXT: v_mov_b32_e32 v1, s4
+; GCN3-NEXT: v_mov_b32_e32 v2, s5
; GCN3-NEXT: .LBB148_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v1, v0
-; GCN3-NEXT: v_subrev_co_u32_e32 v4, vcc, 1, v1
-; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GCN3-NEXT: v_mov_b32_e32 v0, s6
+; GCN3-NEXT: v_mov_b32_e32 v5, v0
+; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v5
+; GCN3-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v2, s4
-; GCN3-NEXT: v_mov_b32_e32 v3, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] offset:16 glc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; GCN3-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB148_1
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
index 0d8f57a9e42bf..3d800c1967ca9 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_noprivate.ll
@@ -455,29 +455,27 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_and_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s3, v3
; GFX7-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB8_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -485,29 +483,27 @@ define amdgpu_kernel void @atomic_and_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_and_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s3, v3
; GFX8-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB8_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -533,70 +529,66 @@ define amdgpu_kernel void @atomic_and_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB9_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_and_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB9_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_offset:
@@ -631,26 +623,24 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s3, v3
; GFX7-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB10_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -665,26 +655,24 @@ define amdgpu_kernel void @atomic_and_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s3, v3
; GFX8-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB10_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -723,30 +711,28 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB11_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_and_i64_ret_addr64_offset:
@@ -760,30 +746,28 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB11_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64_offset:
@@ -819,13 +803,13 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s3, v3
; GFX7-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -847,13 +831,13 @@ define amdgpu_kernel void @atomic_and_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s3, v3
; GFX8-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -892,19 +876,19 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_and_b32_e32 v5, s5, v7
+; GFX7-NEXT: v_and_b32_e32 v4, s4, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB13_1
@@ -925,19 +909,19 @@ define amdgpu_kernel void @atomic_and_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_and_b32_e32 v5, s5, v7
+; GFX8-NEXT: v_and_b32_e32 v4, s4, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB13_1
@@ -977,26 +961,24 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s3, v3
; GFX7-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB14_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -1009,26 +991,24 @@ define amdgpu_kernel void @atomic_and_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s3, v3
; GFX8-NEXT: v_and_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB14_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -1064,30 +1044,28 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB15_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_and_i64_ret_addr64:
@@ -1099,30 +1077,28 @@ define amdgpu_kernel void @atomic_and_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_and_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_and_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_and_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB15_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_and_i64_ret_addr64:
@@ -1151,30 +1127,28 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_sub_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
; GFX7-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s2, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB16_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -1182,30 +1156,28 @@ define amdgpu_kernel void @atomic_sub_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_sub_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
; GFX8-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s2, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB16_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -1231,72 +1203,68 @@ define amdgpu_kernel void @atomic_sub_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
; GFX7-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v2
+; GFX7-NEXT: v_mov_b32_e32 v8, v3
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB17_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_sub_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB17_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_offset:
@@ -1331,27 +1299,25 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
; GFX7-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s2, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB18_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -1366,27 +1332,25 @@ define amdgpu_kernel void @atomic_sub_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s2, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB18_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -1425,31 +1389,29 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
; GFX7-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v2
+; GFX7-NEXT: v_mov_b32_e32 v8, v3
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB19_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_sub_i64_ret_addr64_offset:
@@ -1463,31 +1425,29 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB19_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64_offset:
@@ -1523,14 +1483,14 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s2, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -1552,14 +1512,14 @@ define amdgpu_kernel void @atomic_sub_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s2, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -1598,20 +1558,20 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v0
+; GFX7-NEXT: v_mov_b32_e32 v8, v1
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB21_1
@@ -1632,20 +1592,20 @@ define amdgpu_kernel void @atomic_sub_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
@@ -1685,27 +1645,25 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
; GFX7-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s2, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB22_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -1718,27 +1676,25 @@ define amdgpu_kernel void @atomic_sub_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s2, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -1774,31 +1730,29 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
; GFX7-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v2
+; GFX7-NEXT: v_mov_b32_e32 v8, v3
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s4, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB23_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_sub_i64_ret_addr64:
@@ -1810,31 +1764,29 @@ define amdgpu_kernel void @atomic_sub_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v2
+; GFX8-NEXT: v_mov_b32_e32 v8, v3
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s4, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[7:8]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_sub_i64_ret_addr64:
@@ -1863,31 +1815,29 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_max_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -1895,31 +1845,29 @@ define amdgpu_kernel void @atomic_max_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_max_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB24_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -1945,74 +1893,70 @@ define amdgpu_kernel void @atomic_max_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB25_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_max_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB25_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_offset:
@@ -2047,28 +1991,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB26_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -2083,28 +2025,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -2143,32 +2083,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB27_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -2182,32 +2120,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -2243,16 +2179,16 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -2273,16 +2209,16 @@ define amdgpu_kernel void @atomic_max_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -2320,21 +2256,21 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB29_1
@@ -2355,21 +2291,21 @@ define amdgpu_kernel void @atomic_max_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB29_1
@@ -2409,28 +2345,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB30_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -2443,28 +2377,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -2500,32 +2432,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB31_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_max_i64_ret_addr64:
@@ -2537,32 +2467,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_max_i64_ret_addr64:
@@ -2591,31 +2519,29 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_umax_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB32_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -2623,31 +2549,29 @@ define amdgpu_kernel void @atomic_umax_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_umax_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -2673,74 +2597,70 @@ define amdgpu_kernel void @atomic_umax_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB33_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umax_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_offset:
@@ -2775,28 +2695,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB34_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -2811,28 +2729,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB34_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -2871,32 +2787,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -2910,32 +2824,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -2971,16 +2883,16 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3001,16 +2913,16 @@ define amdgpu_kernel void @atomic_umax_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3048,21 +2960,21 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB37_1
@@ -3083,21 +2995,21 @@ define amdgpu_kernel void @atomic_umax_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB37_1
@@ -3137,28 +3049,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB38_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -3171,28 +3081,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB38_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -3228,32 +3136,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB39_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umax_i64_ret_addr64:
@@ -3265,32 +3171,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB39_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB39_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umax_i64_ret_addr64:
@@ -3319,31 +3223,29 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_min_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB40_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -3351,31 +3253,29 @@ define amdgpu_kernel void @atomic_min_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_min_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB40_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB40_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -3401,74 +3301,70 @@ define amdgpu_kernel void @atomic_min_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB41_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_min_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB41_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB41_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_offset:
@@ -3503,28 +3399,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB42_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -3539,28 +3433,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB42_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -3599,32 +3491,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB43_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -3638,32 +3528,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB43_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -3699,16 +3587,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3729,16 +3617,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3776,21 +3664,21 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB45_1
@@ -3811,21 +3699,21 @@ define amdgpu_kernel void @atomic_min_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB45_1
@@ -3865,28 +3753,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB46_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -3899,28 +3785,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -3956,32 +3840,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_min_i64_ret_addr64:
@@ -3993,32 +3875,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_min_i64_ret_addr64:
@@ -4047,31 +3927,29 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_umin_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB48_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -4079,31 +3957,29 @@ define amdgpu_kernel void @atomic_umin_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_umin_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB48_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -4129,74 +4005,70 @@ define amdgpu_kernel void @atomic_umin_i64_ret_offset(ptr %out, ptr %out2, i64 %
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB49_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umin_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_offset:
@@ -4231,28 +4103,26 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB50_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -4267,28 +4137,26 @@ define amdgpu_kernel void @atomic_umin_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -4327,32 +4195,30 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB51_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umin_i64_ret_addr64_offset:
@@ -4366,32 +4232,30 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64_offset:
@@ -4427,16 +4291,16 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4457,16 +4321,16 @@ define amdgpu_kernel void @atomic_umin_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4504,21 +4368,21 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB53_1
@@ -4539,21 +4403,21 @@ define amdgpu_kernel void @atomic_umin_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
@@ -4593,28 +4457,26 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB54_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -4627,28 +4489,26 @@ define amdgpu_kernel void @atomic_umin_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -4684,32 +4544,30 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umin_i64_ret_addr64:
@@ -4721,32 +4579,30 @@ define amdgpu_kernel void @atomic_umin_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_umin_i64_ret_addr64:
@@ -4775,29 +4631,27 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_or_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s3, v3
; GFX7-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -4805,29 +4659,27 @@ define amdgpu_kernel void @atomic_or_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_or_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s3, v3
; GFX8-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -4853,70 +4705,66 @@ define amdgpu_kernel void @atomic_or_i64_ret_offset(ptr %out, ptr %out2, i64 %in
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_or_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_offset:
@@ -4951,26 +4799,24 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s3, v3
; GFX7-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB58_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -4985,26 +4831,24 @@ define amdgpu_kernel void @atomic_or_i64_addr64_offset(ptr %out, i64 %in, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s3, v3
; GFX8-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -5043,30 +4887,28 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB59_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_or_i64_ret_addr64_offset:
@@ -5080,30 +4922,28 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64_offset:
@@ -5139,13 +4979,13 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s3, v3
; GFX7-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -5167,13 +5007,13 @@ define amdgpu_kernel void @atomic_or_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s3, v3
; GFX8-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -5212,19 +5052,19 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_or_b32_e32 v5, s5, v7
+; GFX7-NEXT: v_or_b32_e32 v4, s4, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB61_1
@@ -5245,19 +5085,19 @@ define amdgpu_kernel void @atomic_or_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_or_b32_e32 v5, s5, v7
+; GFX8-NEXT: v_or_b32_e32 v4, s4, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
@@ -5297,26 +5137,24 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s3, v3
; GFX7-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB62_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -5329,26 +5167,24 @@ define amdgpu_kernel void @atomic_or_i64_addr64(ptr %out, i64 %in, i64 %index) {
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB62_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s3, v3
; GFX8-NEXT: v_or_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB62_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -5384,30 +5220,28 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB63_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_or_i64_ret_addr64:
@@ -5419,30 +5253,28 @@ define amdgpu_kernel void @atomic_or_i64_ret_addr64(ptr %out, ptr %out2, i64 %in
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB63_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_or_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_or_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_or_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB63_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_or_i64_ret_addr64:
@@ -6013,29 +5845,27 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_xor_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB74_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -6043,29 +5873,27 @@ define amdgpu_kernel void @atomic_xor_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_xor_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB74_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -6091,70 +5919,66 @@ define amdgpu_kernel void @atomic_xor_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB75_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_xor_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB75_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_offset:
@@ -6189,26 +6013,24 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB76_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -6223,26 +6045,24 @@ define amdgpu_kernel void @atomic_xor_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB76_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -6281,30 +6101,28 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB77_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_xor_i64_ret_addr64_offset:
@@ -6318,30 +6136,28 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB77_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64_offset:
@@ -6377,13 +6193,13 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -6405,13 +6221,13 @@ define amdgpu_kernel void @atomic_xor_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -6450,19 +6266,19 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
; GFX7-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_xor_b32_e32 v5, s5, v7
+; GFX7-NEXT: v_xor_b32_e32 v4, s4, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB79_1
@@ -6483,19 +6299,19 @@ define amdgpu_kernel void @atomic_xor_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_xor_b32_e32 v5, s5, v7
+; GFX8-NEXT: v_xor_b32_e32 v4, s4, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB79_1
@@ -6535,26 +6351,24 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB80_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -6567,26 +6381,24 @@ define amdgpu_kernel void @atomic_xor_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s3, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s2, v2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB80_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -6622,30 +6434,28 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX7-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB81_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_xor_i64_ret_addr64:
@@ -6657,30 +6467,28 @@ define amdgpu_kernel void @atomic_xor_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_xor_b32_e32 v1, s5, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s4, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, s5, v5
+; GFX8-NEXT: v_xor_b32_e32 v2, s4, v4
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB81_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_xor_i64_ret_addr64:
@@ -8077,22 +7885,20 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_inc_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB107_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8100,9 +7906,9 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB107_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -8110,22 +7916,20 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_inc_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB107_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8133,9 +7937,9 @@ define amdgpu_kernel void @atomic_inc_i64_offset(ptr %out, i64 %in) {
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB107_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -8161,76 +7965,72 @@ define amdgpu_kernel void @atomic_inc_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB108_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB108_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_inc_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB108_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB108_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_offset:
@@ -8265,19 +8065,17 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB109_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8285,9 +8083,9 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB109_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -8302,19 +8100,17 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB109_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8322,9 +8118,9 @@ define amdgpu_kernel void @atomic_inc_i64_incr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB109_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -8363,33 +8159,31 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB110_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB110_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_inc_i64_ret_incr64_offset:
@@ -8403,33 +8197,31 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB110_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB110_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64_offset:
@@ -8465,14 +8257,14 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8496,14 +8288,14 @@ define amdgpu_kernel void @atomic_inc_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8542,32 +8334,30 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB112_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_inc_i64_ret:
@@ -8578,32 +8368,30 @@ define amdgpu_kernel void @atomic_inc_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB112_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret:
@@ -8635,19 +8423,17 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8655,9 +8441,9 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB113_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -8670,19 +8456,17 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8690,9 +8474,9 @@ define amdgpu_kernel void @atomic_inc_i64_incr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB113_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -8728,33 +8512,31 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_mov_b32_e32 v5, v3
+; GFX7-NEXT: v_add_i32_e32 v2, vcc, 1, v4
+; GFX7-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB114_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_inc_i64_ret_incr64:
@@ -8766,33 +8548,31 @@ define amdgpu_kernel void @atomic_inc_i64_ret_incr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
; GFX8-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v4
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB114_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_inc_i64_ret_incr64:
@@ -8821,25 +8601,23 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_dec_i64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_add_u32 s4, s0, 32
-; GFX7-NEXT: s_addc_u32 s5, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, 32
+; GFX7-NEXT: s_addc_u32 s1, s1, 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB115_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s3
-; GFX7-NEXT: v_mov_b32_e32 v7, s2
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[0:1], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8847,9 +8625,9 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB115_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -8857,25 +8635,23 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GFX8-LABEL: atomic_dec_i64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s4, s0, 32
-; GFX8-NEXT: s_addc_u32 s5, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, 32
+; GFX8-NEXT: s_addc_u32 s1, s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB115_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s3
-; GFX8-NEXT: v_mov_b32_e32 v7, s2
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[0:1], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8883,9 +8659,9 @@ define amdgpu_kernel void @atomic_dec_i64_offset(ptr %out, i64 %in) {
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB115_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -8911,82 +8687,78 @@ define amdgpu_kernel void @atomic_dec_i64_ret_offset(ptr %out, ptr %out2, i64 %i
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_add_u32 s6, s0, 32
-; GFX7-NEXT: s_addc_u32 s7, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, 32
+; GFX7-NEXT: s_addc_u32 s1, s1, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB116_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[0:1], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v2, s[0:1], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_mov_b32_e32 v5, s7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB116_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_dec_i64_ret_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[8:9], 0
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s6, s0, 32
-; GFX8-NEXT: s_addc_u32 s7, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, 32
+; GFX8-NEXT: s_addc_u32 s1, s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB116_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v2, s[0:1], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB116_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_offset:
@@ -9019,24 +8791,22 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: s_add_u32 s4, s0, 32
-; GFX7-NEXT: s_addc_u32 s5, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, 32
+; GFX7-NEXT: s_addc_u32 s1, s1, 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB117_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s3
-; GFX7-NEXT: v_mov_b32_e32 v7, s2
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[0:1], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9044,9 +8814,9 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB117_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -9059,24 +8829,22 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: s_add_u32 s4, s0, 32
-; GFX8-NEXT: s_addc_u32 s5, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, 32
+; GFX8-NEXT: s_addc_u32 s1, s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB117_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s3
-; GFX8-NEXT: v_mov_b32_e32 v7, s2
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[0:1], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9084,9 +8852,9 @@ define amdgpu_kernel void @atomic_dec_i64_decr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB117_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -9117,87 +8885,83 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64_offset(ptr %out, ptr %out2,
; GFX7-LABEL: atomic_dec_i64_ret_decr64_offset:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s6
; GFX7-NEXT: s_addc_u32 s1, s1, s7
-; GFX7-NEXT: s_add_u32 s6, s0, 32
-; GFX7-NEXT: s_addc_u32 s7, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, 32
+; GFX7-NEXT: s_addc_u32 s1, s1, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB118_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[0:1], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v2, s[0:1], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_mov_b32_e32 v5, s7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB118_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_dec_i64_ret_decr64_offset:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s6
; GFX8-NEXT: s_addc_u32 s1, s1, s7
-; GFX8-NEXT: s_add_u32 s6, s0, 32
-; GFX8-NEXT: s_addc_u32 s7, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, 32
+; GFX8-NEXT: s_addc_u32 s1, s1, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB118_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v2, s[0:1], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB118_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64_offset:
@@ -9226,24 +8990,24 @@ entry:
define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GFX7-LABEL: atomic_dec_i64:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[2:3], 0
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s7
-; GFX7-NEXT: v_mov_b32_e32 v7, s6
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[0:1], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9251,33 +9015,33 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB119_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_dec_i64:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[2:3], 0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB119_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[0:1], s[6:7], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: v_mov_b32_e32 v7, s6
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[0:1], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9285,9 +9049,9 @@ define amdgpu_kernel void @atomic_dec_i64(ptr %out, i64 %in) {
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB119_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -9310,80 +9074,76 @@ entry:
define amdgpu_kernel void @atomic_dec_i64_ret(ptr %out, ptr %out2, i64 %in) {
; GFX7-LABEL: atomic_dec_i64_ret:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0xd
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[0:1], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v2, s[0:1], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s8
-; GFX7-NEXT: v_mov_b32_e32 v5, s9
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB120_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v2, s10
-; GFX7-NEXT: v_mov_b32_e32 v3, s11
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_dec_i64_ret:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB120_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v2, s[0:1], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB120_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, s10
-; GFX8-NEXT: v_mov_b32_e32 v3, s11
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret:
@@ -9413,24 +9173,22 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GFX7-NEXT: s_add_u32 s4, s0, s4
-; GFX7-NEXT: s_addc_u32 s5, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, s4
+; GFX7-NEXT: s_addc_u32 s1, s1, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s3
-; GFX7-NEXT: v_mov_b32_e32 v7, s2
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[0:1], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9438,9 +9196,9 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX7-NEXT: s_cbranch_execnz .LBB121_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -9451,24 +9209,22 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; GFX8-NEXT: s_add_u32 s4, s0, s4
-; GFX8-NEXT: s_addc_u32 s5, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, s4
+; GFX8-NEXT: s_addc_u32 s1, s1, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[0:1], s[2:3], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s3
-; GFX8-NEXT: v_mov_b32_e32 v7, s2
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[0:1], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
+; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -9476,9 +9232,9 @@ define amdgpu_kernel void @atomic_dec_i64_decr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB121_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -9508,83 +9264,79 @@ define amdgpu_kernel void @atomic_dec_i64_ret_decr64(ptr %out, ptr %out2, i64 %i
; GFX7-LABEL: atomic_dec_i64_ret_decr64:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; GFX7-NEXT: s_add_u32 s6, s0, s6
-; GFX7-NEXT: s_addc_u32 s7, s1, s7
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX7-NEXT: s_add_u32 s0, s0, s6
+; GFX7-NEXT: s_addc_u32 s1, s1, s7
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[0:1], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v2, s[0:1], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX7-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_mov_b32_e32 v5, s7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB122_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_dec_i64_ret_decr64:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b64 s[8:9], 0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; GFX8-NEXT: s_add_u32 s6, s0, s6
-; GFX8-NEXT: s_addc_u32 s7, s1, s7
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1] glc
+; GFX8-NEXT: s_add_u32 s0, s0, s6
+; GFX8-NEXT: s_addc_u32 s1, s1, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[0:1], 0, v3, s[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v2, s[0:1], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v9, s[0:1]
; GFX8-NEXT: s_or_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v5, s7
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v2, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB122_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX12-LABEL: atomic_dec_i64_ret_decr64:
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
index 2d6b544a8a190..94722ee4a3b51 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system.ll
@@ -4493,14 +4493,12 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: .LBB34_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
+; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -4558,14 +4556,12 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: .LBB34_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
+; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -4611,18 +4607,16 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: .LBB34_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB34_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: .LBB34_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4683,14 +4677,12 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
; GCN1-NEXT: .LBB35_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
+; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -4750,14 +4742,12 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
; GCN2-NEXT: .LBB35_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
+; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -4805,18 +4795,16 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB35_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB35_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
; GCN3-NEXT: .LBB35_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4870,20 +4858,18 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: .LBB36_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v1, s7
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v0
+; GCN1-NEXT: v_mov_b32_e32 v8, v1
+; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
+; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB36_2
@@ -4933,20 +4919,18 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: .LBB36_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v1, s7
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v0
+; GCN2-NEXT: v_mov_b32_e32 v8, v1
+; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB36_2
@@ -4984,24 +4968,22 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB36_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: .LBB36_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
-; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v0
+; GCN3-NEXT: v_mov_b32_e32 v8, v1
+; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB36_2
@@ -5054,20 +5036,18 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
; GCN1-NEXT: .LBB37_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v1, s7
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v0
+; GCN1-NEXT: v_mov_b32_e32 v8, v1
+; GCN1-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
+; GCN1-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB37_2
@@ -5119,20 +5099,18 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
; GCN2-NEXT: .LBB37_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v1, s7
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v0
+; GCN2-NEXT: v_mov_b32_e32 v8, v1
+; GCN2-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; GCN2-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB37_2
@@ -5172,24 +5150,22 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB37_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
; GCN3-NEXT: .LBB37_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
-; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v0
+; GCN3-NEXT: v_mov_b32_e32 v8, v1
+; GCN3-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GCN3-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB37_2
@@ -6233,8 +6209,6 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -6296,8 +6270,6 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -6342,17 +6314,15 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: .LBB44_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB44_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB44_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -6417,8 +6387,6 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -6482,8 +6450,6 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -6530,17 +6496,15 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB45_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB45_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB45_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -6596,16 +6560,14 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: .LBB46_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB46_2
@@ -6657,16 +6619,14 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: .LBB46_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB46_2
@@ -6703,23 +6663,21 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB46_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB46_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB46_2
@@ -6774,16 +6732,14 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: .LBB47_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB47_2
@@ -6837,16 +6793,14 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: .LBB47_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB47_2
@@ -6885,23 +6839,21 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB47_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB47_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_and_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB47_2
@@ -7992,8 +7944,6 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8059,8 +8009,6 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8109,17 +8057,15 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: .LBB54_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB54_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB54_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8188,8 +8134,6 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: v_not_b32_e32 v1, v0
; GCN1-NEXT: v_not_b32_e32 v0, v6
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8257,8 +8201,6 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: v_not_b32_e32 v1, v0
; GCN2-NEXT: v_not_b32_e32 v0, v6
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8309,17 +8251,15 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB55_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB55_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB55_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: v_not_b32_e32 v1, v0
; GCN3-NEXT: v_not_b32_e32 v0, v6
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -8379,18 +8319,16 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: .LBB56_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_not_b32_e32 v1, v0
-; GCN1-NEXT: v_not_b32_e32 v0, v6
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN1-NEXT: v_not_b32_e32 v5, v0
+; GCN1-NEXT: v_not_b32_e32 v4, v1
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB56_2
@@ -8444,18 +8382,16 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: .LBB56_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_not_b32_e32 v1, v0
-; GCN2-NEXT: v_not_b32_e32 v0, v6
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN2-NEXT: v_not_b32_e32 v5, v0
+; GCN2-NEXT: v_not_b32_e32 v4, v1
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB56_2
@@ -8494,25 +8430,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB56_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB56_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_not_b32_e32 v1, v0
-; GCN3-NEXT: v_not_b32_e32 v0, v6
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN3-NEXT: v_not_b32_e32 v5, v0
+; GCN3-NEXT: v_not_b32_e32 v4, v1
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB56_2
@@ -8569,18 +8503,16 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: .LBB57_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN1-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_not_b32_e32 v1, v0
-; GCN1-NEXT: v_not_b32_e32 v0, v6
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN1-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN1-NEXT: v_not_b32_e32 v5, v0
+; GCN1-NEXT: v_not_b32_e32 v4, v1
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB57_2
@@ -8636,18 +8568,16 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: .LBB57_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN2-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_not_b32_e32 v1, v0
-; GCN2-NEXT: v_not_b32_e32 v0, v6
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN2-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN2-NEXT: v_not_b32_e32 v5, v0
+; GCN2-NEXT: v_not_b32_e32 v4, v1
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB57_2
@@ -8688,25 +8618,23 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB57_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB57_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_and_b32_e32 v0, s7, v3
-; GCN3-NEXT: v_and_b32_e32 v6, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_not_b32_e32 v1, v0
-; GCN3-NEXT: v_not_b32_e32 v0, v6
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_and_b32_e32 v0, s7, v7
+; GCN3-NEXT: v_and_b32_e32 v1, s6, v6
+; GCN3-NEXT: v_not_b32_e32 v5, v0
+; GCN3-NEXT: v_not_b32_e32 v4, v1
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB57_2
@@ -9883,8 +9811,6 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -9946,8 +9872,6 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -9992,17 +9916,15 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GCN3-NEXT: .LBB64_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB64_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB64_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -10067,8 +9989,6 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -10132,8 +10052,6 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -10180,17 +10098,15 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GCN3-NEXT: .LBB65_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB65_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB65_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -10246,16 +10162,14 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN1-NEXT: .LBB66_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB66_2
@@ -10307,16 +10221,14 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN2-NEXT: .LBB66_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB66_2
@@ -10353,23 +10265,21 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB66_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB66_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB66_2
@@ -10424,16 +10334,14 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: .LBB67_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB67_2
@@ -10487,16 +10395,14 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: .LBB67_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB67_2
@@ -10535,23 +10441,21 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB67_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB67_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_or_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_or_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_or_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_or_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB67_2
@@ -11594,8 +11498,6 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -11657,8 +11559,6 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -11703,17 +11603,15 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: .LBB74_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB74_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB74_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -11778,8 +11676,6 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -11843,8 +11739,6 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -11891,17 +11785,15 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB75_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB75_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB75_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -11957,16 +11849,14 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: .LBB76_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB76_2
@@ -12018,16 +11908,14 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: .LBB76_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB76_2
@@ -12064,23 +11952,21 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB76_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB76_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB76_2
@@ -12135,16 +12021,14 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: .LBB77_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN1-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN1-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB77_2
@@ -12198,16 +12082,14 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: .LBB77_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN2-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN2-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB77_2
@@ -12246,23 +12128,21 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB77_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB77_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_xor_b32_e32 v1, s7, v3
-; GCN3-NEXT: v_xor_b32_e32 v0, s6, v2
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_xor_b32_e32 v5, s7, v7
+; GCN3-NEXT: v_xor_b32_e32 v4, s6, v6
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB77_2
@@ -13300,16 +13180,14 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB84_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -13368,16 +13246,14 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB84_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -13424,20 +13300,18 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: .LBB84_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB84_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB84_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -13499,16 +13373,14 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB85_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -13569,16 +13441,14 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB85_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -13627,20 +13497,18 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB85_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB85_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB85_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -13695,22 +13563,20 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB86_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB86_2
@@ -13761,22 +13627,20 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB86_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB86_2
@@ -13815,26 +13679,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB86_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB86_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB86_2
@@ -13888,22 +13750,20 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB87_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB87_2
@@ -13956,22 +13816,20 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB87_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB87_2
@@ -14012,26 +13870,24 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB87_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB87_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB87_2
@@ -14092,20 +13948,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: .LBB88_2: ; %atomicrmw.phi
; GCN1-NEXT: s_endpgm
; GCN1-NEXT: .LBB88_3: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB88_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -14166,20 +14020,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2-NEXT: .LBB88_2: ; %atomicrmw.phi
; GCN2-NEXT: s_endpgm
; GCN2-NEXT: .LBB88_3: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB88_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -14238,20 +14090,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: .LBB88_2: ; %atomicrmw.phi
; GCN3-NEXT: s_endpgm
; GCN3-NEXT: .LBB88_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
; GCN3-NEXT: .LBB88_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -14308,26 +14158,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB89_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
-; GCN1-NEXT: .LBB89_2: ; %atomicrmw.start
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
+; GCN1-NEXT: .LBB89_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB89_2
@@ -14381,26 +14229,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB89_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB89_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB89_2
@@ -14453,26 +14299,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB89_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB89_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB89_2
@@ -14536,20 +14380,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN1-NEXT: .LBB90_2: ; %atomicrmw.phi
; GCN1-NEXT: s_endpgm
; GCN1-NEXT: .LBB90_3: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB90_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -14607,20 +14449,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN2-NEXT: .LBB90_2: ; %atomicrmw.phi
; GCN2-NEXT: s_endpgm
; GCN2-NEXT: .LBB90_3: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB90_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -14677,20 +14517,18 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GCN3-NEXT: .LBB90_2: ; %atomicrmw.phi
; GCN3-NEXT: s_endpgm
; GCN3-NEXT: .LBB90_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
; GCN3-NEXT: .LBB90_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -14744,26 +14582,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB91_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
; GCN1-NEXT: .LBB91_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB91_2
@@ -14815,26 +14651,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB91_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB91_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB91_2
@@ -14885,26 +14719,24 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB91_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB91_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB91_2
@@ -15948,16 +15780,14 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB98_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -16016,16 +15846,14 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB98_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -16072,20 +15900,18 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: .LBB98_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB98_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB98_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -16147,16 +15973,14 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB99_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -16217,16 +16041,14 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB99_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -16275,20 +16097,18 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB99_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB99_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB99_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -16343,22 +16163,20 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB100_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB100_2
@@ -16409,22 +16227,20 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB100_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB100_2
@@ -16463,26 +16279,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB100_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB100_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB100_2
@@ -16536,22 +16350,20 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB101_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB101_2
@@ -16604,22 +16416,20 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB101_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB101_2
@@ -16660,26 +16470,24 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB101_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB101_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB101_2
@@ -16740,20 +16548,18 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN1-NEXT: .LBB102_2: ; %atomicrmw.phi
; GCN1-NEXT: s_endpgm
; GCN1-NEXT: .LBB102_3: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB102_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -16814,20 +16620,18 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN2-NEXT: .LBB102_2: ; %atomicrmw.phi
; GCN2-NEXT: s_endpgm
; GCN2-NEXT: .LBB102_3: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB102_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -16886,20 +16690,18 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GCN3-NEXT: .LBB102_2: ; %atomicrmw.phi
; GCN3-NEXT: s_endpgm
; GCN3-NEXT: .LBB102_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
; GCN3-NEXT: .LBB102_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -16956,26 +16758,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB103_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
; GCN1-NEXT: .LBB103_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB103_2
@@ -17029,26 +16829,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB103_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB103_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB103_2
@@ -17101,26 +16899,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB103_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB103_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB103_2
@@ -17177,26 +16973,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB104_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
; GCN1-NEXT: .LBB104_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB104_2
@@ -17248,26 +17042,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB104_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB104_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB104_2
@@ -17318,26 +17110,24 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB104_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB104_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB104_2
@@ -18381,16 +18171,14 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB111_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -18449,16 +18237,14 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB111_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -18505,20 +18291,18 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GCN3-NEXT: .LBB111_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB111_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB111_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -18580,16 +18364,14 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB112_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -18650,16 +18432,14 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB112_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -18708,20 +18488,18 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB112_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB112_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB112_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -18776,22 +18554,20 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB113_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB113_2
@@ -18842,22 +18618,20 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB113_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB113_2
@@ -18896,26 +18670,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB113_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB113_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB113_2
@@ -18969,22 +18741,20 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB114_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB114_2
@@ -19037,22 +18807,20 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB114_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB114_2
@@ -19093,26 +18861,24 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB114_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB114_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB114_2
@@ -20152,16 +19918,14 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB121_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -20220,16 +19984,14 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB121_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -20276,20 +20038,18 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GCN3-NEXT: .LBB121_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB121_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB121_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -20351,16 +20111,14 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB122_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -20421,16 +20179,14 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB122_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -20479,20 +20235,18 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GCN3-NEXT: .LBB122_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB122_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB122_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -20547,22 +20301,20 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[34:35], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB123_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB123_2
@@ -20613,22 +20365,20 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[34:35], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB123_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB123_2
@@ -20667,26 +20417,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB123_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB123_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB123_2
@@ -20740,22 +20488,20 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB124_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB124_2
@@ -20808,22 +20554,20 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB124_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB124_2
@@ -20864,26 +20608,24 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB124_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB124_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB124_2
@@ -20944,20 +20686,18 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN1-NEXT: .LBB125_2: ; %atomicrmw.phi
; GCN1-NEXT: s_endpgm
; GCN1-NEXT: .LBB125_3: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB125_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -21018,20 +20758,18 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN2-NEXT: .LBB125_2: ; %atomicrmw.phi
; GCN2-NEXT: s_endpgm
; GCN2-NEXT: .LBB125_3: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB125_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -21090,20 +20828,18 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GCN3-NEXT: .LBB125_2: ; %atomicrmw.phi
; GCN3-NEXT: s_endpgm
; GCN3-NEXT: .LBB125_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
; GCN3-NEXT: .LBB125_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -21160,26 +20896,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB126_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
; GCN1-NEXT: .LBB126_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB126_2
@@ -21233,26 +20967,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB126_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB126_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB126_2
@@ -21305,26 +21037,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB126_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB126_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB126_2
@@ -21384,20 +21114,18 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN1-NEXT: .LBB127_2: ; %atomicrmw.phi
; GCN1-NEXT: s_endpgm
; GCN1-NEXT: .LBB127_3: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v5, s1
+; GCN1-NEXT: v_mov_b32_e32 v4, s0
+; GCN1-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN1-NEXT: s_mov_b64 s[4:5], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s3
+; GCN1-NEXT: v_mov_b32_e32 v7, s2
; GCN1-NEXT: .LBB127_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s3
-; GCN1-NEXT: v_mov_b32_e32 v6, s2
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -21451,20 +21179,18 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN2-NEXT: .LBB127_2: ; %atomicrmw.phi
; GCN2-NEXT: s_endpgm
; GCN2-NEXT: .LBB127_3: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v5, s1
+; GCN2-NEXT: v_mov_b32_e32 v4, s0
+; GCN2-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN2-NEXT: s_mov_b64 s[4:5], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s3
+; GCN2-NEXT: v_mov_b32_e32 v7, s2
; GCN2-NEXT: .LBB127_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s3
-; GCN2-NEXT: v_mov_b32_e32 v6, s2
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -21517,20 +21243,18 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GCN3-NEXT: .LBB127_2: ; %atomicrmw.phi
; GCN3-NEXT: s_endpgm
; GCN3-NEXT: .LBB127_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v5, s1
+; GCN3-NEXT: v_mov_b32_e32 v4, s0
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[4:5], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s3
+; GCN3-NEXT: v_mov_b32_e32 v7, s2
; GCN3-NEXT: .LBB127_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s3
-; GCN3-NEXT: v_mov_b32_e32 v6, s2
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -21583,26 +21307,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN1-NEXT: s_cmp_lg_u32 s2, 1
; GCN1-NEXT: s_cbranch_scc0 .LBB128_4
; GCN1-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN1-NEXT: s_mov_b64 s[2:3], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s13
+; GCN1-NEXT: v_mov_b32_e32 v5, s12
; GCN1-NEXT: .LBB128_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v0, s13
-; GCN1-NEXT: v_mov_b32_e32 v6, s12
-; GCN1-NEXT: v_mov_b32_e32 v5, s1
-; GCN1-NEXT: v_mov_b32_e32 v4, s0
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN1-NEXT: s_cbranch_execnz .LBB128_2
@@ -21654,26 +21376,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN2-NEXT: s_cmp_lg_u32 s2, 1
; GCN2-NEXT: s_cbranch_scc0 .LBB128_4
; GCN2-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN2-NEXT: s_mov_b64 s[2:3], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s13
+; GCN2-NEXT: v_mov_b32_e32 v5, s12
; GCN2-NEXT: .LBB128_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v0, s13
-; GCN2-NEXT: v_mov_b32_e32 v6, s12
-; GCN2-NEXT: v_mov_b32_e32 v5, s1
-; GCN2-NEXT: v_mov_b32_e32 v4, s0
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN2-NEXT: s_cbranch_execnz .LBB128_2
@@ -21724,26 +21444,24 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GCN3-NEXT: s_cmp_lg_u32 s2, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB128_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v3, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s0
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[2:3], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s13
+; GCN3-NEXT: v_mov_b32_e32 v5, s12
; GCN3-NEXT: .LBB128_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v0, s13
-; GCN3-NEXT: v_mov_b32_e32 v6, s12
-; GCN3-NEXT: v_mov_b32_e32 v5, s1
-; GCN3-NEXT: v_mov_b32_e32 v4, s0
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GCN3-NEXT: s_cbranch_execnz .LBB128_2
@@ -22853,8 +22571,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -22922,8 +22638,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -22973,9 +22687,9 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: .LBB135_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB135_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB135_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -22983,8 +22697,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -23055,8 +22767,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -23126,8 +22836,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -23179,9 +22887,9 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN3-NEXT: .LBB136_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB136_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s34
+; GCN3-NEXT: v_mov_b32_e32 v5, s35
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB136_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -23189,8 +22897,6 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -23251,19 +22957,17 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN1-NEXT: .LBB137_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6
+; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN1-NEXT: s_cbranch_execnz .LBB137_2
@@ -23318,19 +23022,17 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN2-NEXT: .LBB137_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN2-NEXT: s_cbranch_execnz .LBB137_2
@@ -23370,26 +23072,24 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB137_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[34:35], 0
; GCN3-NEXT: .LBB137_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
-; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
+; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GCN3-NEXT: s_cbranch_execnz .LBB137_2
@@ -23447,19 +23147,17 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN1-NEXT: .LBB138_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_mov_b32_e32 v4, s34
-; GCN1-NEXT: v_mov_b32_e32 v5, s35
-; GCN1-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v6, v0
+; GCN1-NEXT: v_mov_b32_e32 v7, v1
+; GCN1-NEXT: v_add_i32_e32 v0, vcc, 1, v6
+; GCN1-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN1-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN1-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB138_2
@@ -23516,19 +23214,17 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN2-NEXT: .LBB138_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_mov_b32_e32 v4, s34
-; GCN2-NEXT: v_mov_b32_e32 v5, s35
-; GCN2-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v6, v0
+; GCN2-NEXT: v_mov_b32_e32 v7, v1
+; GCN2-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; GCN2-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GCN2-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN2-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB138_2
@@ -23570,26 +23266,24 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: s_cmp_lg_u32 s36, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB138_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s34
-; GCN3-NEXT: v_mov_b32_e32 v1, s35
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s34
+; GCN3-NEXT: v_mov_b32_e32 v3, s35
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
; GCN3-NEXT: .LBB138_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
-; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_mov_b32_e32 v4, s34
-; GCN3-NEXT: v_mov_b32_e32 v5, s35
-; GCN3-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_mov_b32_e32 v6, v0
+; GCN3-NEXT: v_mov_b32_e32 v7, v1
+; GCN3-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
+; GCN3-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GCN3-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GCN3-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB138_2
@@ -24726,17 +24420,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB145_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN1-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GCN1-NEXT: v_mov_b32_e32 v6, s7
-; GCN1-NEXT: v_mov_b32_e32 v7, s6
-; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN1-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v2
+; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN1-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -24800,17 +24492,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB145_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN2-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GCN2-NEXT: v_mov_b32_e32 v6, s7
-; GCN2-NEXT: v_mov_b32_e32 v7, s6
-; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN2-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN2-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -24862,21 +24552,19 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GCN3-NEXT: .LBB145_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB145_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s4
+; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB145_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN3-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
-; GCN3-NEXT: v_mov_b32_e32 v6, s7
-; GCN3-NEXT: v_mov_b32_e32 v7, s6
-; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN3-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN3-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -24943,17 +24631,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: flat_load_dword v2, v[4:5]
; GCN1-NEXT: s_mov_b64 s[38:39], 0
+; GCN1-NEXT: v_mov_b32_e32 v6, s7
+; GCN1-NEXT: v_mov_b32_e32 v7, s6
; GCN1-NEXT: .LBB146_4: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GCN1-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN1-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GCN1-NEXT: v_mov_b32_e32 v6, s7
-; GCN1-NEXT: v_mov_b32_e32 v7, s6
-; GCN1-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v4, s36
-; GCN1-NEXT: v_mov_b32_e32 v5, s37
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN1-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v2
+; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN1-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN1-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -25019,17 +24705,15 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: flat_load_dword v2, v[4:5]
; GCN2-NEXT: s_mov_b64 s[38:39], 0
+; GCN2-NEXT: v_mov_b32_e32 v6, s7
+; GCN2-NEXT: v_mov_b32_e32 v7, s6
; GCN2-NEXT: .LBB146_4: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GCN2-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN2-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GCN2-NEXT: v_mov_b32_e32 v6, s7
-; GCN2-NEXT: v_mov_b32_e32 v7, s6
-; GCN2-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v4, s36
-; GCN2-NEXT: v_mov_b32_e32 v5, s37
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN2-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN2-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN2-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -25083,21 +24767,19 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GCN3-NEXT: .LBB146_2: ; %atomicrmw.phi
; GCN3-NEXT: s_setpc_b64 s[30:31]
; GCN3-NEXT: .LBB146_3: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s36
-; GCN3-NEXT: v_mov_b32_e32 v1, s37
-; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v4, s36
+; GCN3-NEXT: v_mov_b32_e32 v5, s37
+; GCN3-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
; GCN3-NEXT: s_mov_b64 s[38:39], 0
+; GCN3-NEXT: v_mov_b32_e32 v6, s7
+; GCN3-NEXT: v_mov_b32_e32 v7, s6
; GCN3-NEXT: .LBB146_4: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GCN3-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GCN3-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
-; GCN3-NEXT: v_mov_b32_e32 v6, s7
-; GCN3-NEXT: v_mov_b32_e32 v7, s6
-; GCN3-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v4, s36
-; GCN3-NEXT: v_mov_b32_e32 v5, s37
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GCN3-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GCN3-NEXT: s_or_b64 vcc, s[34:35], vcc
; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GCN3-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -25157,25 +24839,23 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[36:37], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB147_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_subrev_i32_e64 v7, s[34:35], 1, v2
-; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v8
+; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN1-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN1-NEXT: v_mov_b32_e32 v4, s4
-; GCN1-NEXT: v_mov_b32_e32 v5, s5
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN1-NEXT: s_cbranch_execnz .LBB147_2
@@ -25230,25 +24910,23 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[36:37], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB147_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN2-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN2-NEXT: v_mov_b32_e32 v4, s4
-; GCN2-NEXT: v_mov_b32_e32 v5, s5
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN2-NEXT: s_cbranch_execnz .LBB147_2
@@ -25291,29 +24969,27 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB147_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s4
-; GCN3-NEXT: v_mov_b32_e32 v1, s5
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s4
+; GCN3-NEXT: v_mov_b32_e32 v3, s5
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[36:37], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB147_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_subrev_co_u32_e64 v7, s[34:35], 1, v2
-; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v8
+; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN3-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN3-NEXT: v_mov_b32_e32 v4, s4
-; GCN3-NEXT: v_mov_b32_e32 v5, s5
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GCN3-NEXT: s_cbranch_execnz .LBB147_2
@@ -25371,25 +25047,23 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN1-NEXT: flat_load_dword v1, v[0:1]
; GCN1-NEXT: flat_load_dword v0, v[2:3]
; GCN1-NEXT: s_mov_b64 s[38:39], 0
+; GCN1-NEXT: v_mov_b32_e32 v4, s7
+; GCN1-NEXT: v_mov_b32_e32 v5, s6
; GCN1-NEXT: .LBB148_2: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v3, v1
-; GCN1-NEXT: v_mov_b32_e32 v2, v0
-; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN1-NEXT: v_subrev_i32_e64 v7, s[34:35], 1, v2
-; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN1-NEXT: v_mov_b32_e32 v0, s7
-; GCN1-NEXT: v_mov_b32_e32 v6, s6
+; GCN1-NEXT: v_mov_b32_e32 v9, v1
+; GCN1-NEXT: v_mov_b32_e32 v8, v0
+; GCN1-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN1-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v8
+; GCN1-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN1-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN1-NEXT: v_mov_b32_e32 v4, s36
-; GCN1-NEXT: v_mov_b32_e32 v5, s37
-; GCN1-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN1-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN1-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN1-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN1-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN1-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN1-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
; GCN1-NEXT: s_andn2_b64 exec, exec, s[38:39]
; GCN1-NEXT: s_cbranch_execnz .LBB148_2
@@ -25446,25 +25120,23 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN2-NEXT: flat_load_dword v1, v[0:1]
; GCN2-NEXT: flat_load_dword v0, v[2:3]
; GCN2-NEXT: s_mov_b64 s[38:39], 0
+; GCN2-NEXT: v_mov_b32_e32 v4, s7
+; GCN2-NEXT: v_mov_b32_e32 v5, s6
; GCN2-NEXT: .LBB148_2: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v3, v1
-; GCN2-NEXT: v_mov_b32_e32 v2, v0
-; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN2-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN2-NEXT: v_mov_b32_e32 v0, s7
-; GCN2-NEXT: v_mov_b32_e32 v6, s6
+; GCN2-NEXT: v_mov_b32_e32 v9, v1
+; GCN2-NEXT: v_mov_b32_e32 v8, v0
+; GCN2-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN2-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; GCN2-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN2-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN2-NEXT: v_mov_b32_e32 v4, s36
-; GCN2-NEXT: v_mov_b32_e32 v5, s37
-; GCN2-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN2-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN2-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN2-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN2-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN2-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN2-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
; GCN2-NEXT: s_andn2_b64 exec, exec, s[38:39]
; GCN2-NEXT: s_cbranch_execnz .LBB148_2
@@ -25509,29 +25181,27 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GCN3-NEXT: s_cmp_lg_u32 s34, 1
; GCN3-NEXT: s_cbranch_scc0 .LBB148_4
; GCN3-NEXT: ; %bb.1: ; %atomicrmw.global
-; GCN3-NEXT: v_mov_b32_e32 v0, s36
-; GCN3-NEXT: v_mov_b32_e32 v1, s37
-; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GCN3-NEXT: v_mov_b32_e32 v2, s36
+; GCN3-NEXT: v_mov_b32_e32 v3, s37
+; GCN3-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
; GCN3-NEXT: s_mov_b64 s[38:39], 0
+; GCN3-NEXT: v_mov_b32_e32 v4, s7
+; GCN3-NEXT: v_mov_b32_e32 v5, s6
; GCN3-NEXT: .LBB148_2: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v3, v1
-; GCN3-NEXT: v_mov_b32_e32 v2, v0
-; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GCN3-NEXT: v_subrev_co_u32_e64 v7, s[34:35], 1, v2
-; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GCN3-NEXT: v_mov_b32_e32 v0, s7
-; GCN3-NEXT: v_mov_b32_e32 v6, s6
+; GCN3-NEXT: v_mov_b32_e32 v9, v1
+; GCN3-NEXT: v_mov_b32_e32 v8, v0
+; GCN3-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GCN3-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v8
+; GCN3-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GCN3-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GCN3-NEXT: v_mov_b32_e32 v4, s36
-; GCN3-NEXT: v_mov_b32_e32 v5, s37
-; GCN3-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GCN3-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GCN3-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GCN3-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GCN3-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GCN3-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GCN3-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
; GCN3-NEXT: s_andn2_b64 exec, exec, s[38:39]
; GCN3-NEXT: s_cbranch_execnz .LBB148_2
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
index 4305c25070d79..63d001b635cf2 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i64_system_noprivate.ll
@@ -1440,14 +1440,14 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -1473,14 +1473,14 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -1501,14 +1501,14 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1539,26 +1539,24 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
; GFX7-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
+; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB35_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
@@ -1574,26 +1572,24 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
+; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_sub_i64_noret_offset_scalar:
@@ -1603,14 +1599,14 @@ define amdgpu_gfx void @flat_atomic_sub_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v6, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1641,20 +1637,20 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v0
+; GFX7-NEXT: v_mov_b32_e32 v8, v1
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB36_1
@@ -1674,20 +1670,20 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB36_1
@@ -1702,20 +1698,20 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB36_1
@@ -1740,26 +1736,24 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
; GFX7-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v0
+; GFX7-NEXT: v_mov_b32_e32 v8, v1
+; GFX7-NEXT: v_subrev_i32_e32 v5, vcc, s6, v7
+; GFX7-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB37_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
@@ -1775,26 +1769,24 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
; GFX8-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v0
+; GFX8-NEXT: v_mov_b32_e32 v8, v1
+; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB37_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_sub_i64_ret_offset_scalar:
@@ -1804,20 +1796,20 @@ define amdgpu_gfx i64 @flat_atomic_sub_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v5, vcc, s6, v7
+; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v8, v4, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB37_1
@@ -2246,13 +2238,13 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s7, v3
; GFX7-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -2278,13 +2270,13 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s7, v3
; GFX8-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -2305,13 +2297,13 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB44_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, s7, v3
; GFX9-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -2342,25 +2334,23 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v1, s7, v3
; GFX7-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB45_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_and_i64_noret_offset_scalar:
@@ -2376,25 +2366,23 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v1, s7, v3
; GFX8-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB45_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_and_i64_noret_offset_scalar:
@@ -2404,13 +2392,13 @@ define amdgpu_gfx void @flat_atomic_and_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB45_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v1, s7, v3
; GFX9-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -2441,19 +2429,19 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB46_1
@@ -2473,19 +2461,19 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
@@ -2500,19 +2488,19 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB46_1
@@ -2537,25 +2525,23 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB47_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_and_i64_ret_offset_scalar:
@@ -2571,25 +2557,23 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_and_i64_ret_offset_scalar:
@@ -2599,19 +2583,19 @@ define amdgpu_gfx i64 @flat_atomic_and_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_and_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_and_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_and_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_and_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB47_1
@@ -3064,13 +3048,13 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: v_not_b32_e32 v1, v0
; GFX7-NEXT: v_not_b32_e32 v0, v6
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3098,13 +3082,13 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: v_not_b32_e32 v1, v0
; GFX8-NEXT: v_not_b32_e32 v0, v6
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3127,13 +3111,13 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_not_b32_e32 v1, v0
; GFX9-NEXT: v_not_b32_e32 v0, v6
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3166,14 +3150,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: v_not_b32_e32 v1, v0
; GFX7-NEXT: v_not_b32_e32 v0, v6
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3181,12 +3163,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB55_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
@@ -3202,14 +3184,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: v_not_b32_e32 v1, v0
; GFX8-NEXT: v_not_b32_e32 v0, v6
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3217,12 +3197,12 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_nand_i64_noret_offset_scalar:
@@ -3232,13 +3212,13 @@ define amdgpu_gfx void @flat_atomic_nand_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_not_b32_e32 v1, v0
; GFX9-NEXT: v_not_b32_e32 v0, v6
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
@@ -3271,21 +3251,21 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_not_b32_e32 v1, v0
-; GFX7-NEXT: v_not_b32_e32 v0, v6
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX7-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX7-NEXT: v_not_b32_e32 v5, v0
+; GFX7-NEXT: v_not_b32_e32 v4, v1
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB56_1
@@ -3305,21 +3285,21 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_not_b32_e32 v1, v0
-; GFX8-NEXT: v_not_b32_e32 v0, v6
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX8-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX8-NEXT: v_not_b32_e32 v5, v0
+; GFX8-NEXT: v_not_b32_e32 v4, v1
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -3334,21 +3314,21 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_not_b32_e32 v1, v0
-; GFX9-NEXT: v_not_b32_e32 v0, v6
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX9-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX9-NEXT: v_not_b32_e32 v5, v0
+; GFX9-NEXT: v_not_b32_e32 v4, v1
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB56_1
@@ -3373,27 +3353,25 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX7-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_not_b32_e32 v1, v0
-; GFX7-NEXT: v_not_b32_e32 v0, v6
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX7-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX7-NEXT: v_not_b32_e32 v5, v0
+; GFX7-NEXT: v_not_b32_e32 v4, v1
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB57_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
@@ -3409,27 +3387,25 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX8-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_not_b32_e32 v1, v0
-; GFX8-NEXT: v_not_b32_e32 v0, v6
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX8-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX8-NEXT: v_not_b32_e32 v5, v0
+; GFX8-NEXT: v_not_b32_e32 v4, v1
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_nand_i64_ret_offset_scalar:
@@ -3439,21 +3415,21 @@ define amdgpu_gfx i64 @flat_atomic_nand_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_and_b32_e32 v0, s7, v3
-; GFX9-NEXT: v_and_b32_e32 v6, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_not_b32_e32 v1, v0
-; GFX9-NEXT: v_not_b32_e32 v0, v6
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_and_b32_e32 v0, s7, v7
+; GFX9-NEXT: v_and_b32_e32 v1, s6, v6
+; GFX9-NEXT: v_not_b32_e32 v5, v0
+; GFX9-NEXT: v_not_b32_e32 v4, v1
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB57_1
@@ -3998,13 +3974,13 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB64_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s7, v3
; GFX7-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -4030,13 +4006,13 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB64_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s7, v3
; GFX8-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -4057,13 +4033,13 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_scalar(ptr inreg %ptr, i64 inre
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB64_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v1, s7, v3
; GFX9-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4094,25 +4070,23 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB65_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_or_b32_e32 v1, s7, v3
; GFX7-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB65_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_or_i64_noret_offset_scalar:
@@ -4128,25 +4102,23 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB65_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_or_b32_e32 v1, s7, v3
; GFX8-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB65_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_or_i64_noret_offset_scalar:
@@ -4156,13 +4128,13 @@ define amdgpu_gfx void @flat_atomic_or_i64_noret_offset_scalar(ptr inreg %out, i
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB65_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v1, s7, v3
; GFX9-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4193,19 +4165,19 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB66_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB66_1
@@ -4225,19 +4197,19 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB66_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB66_1
@@ -4252,19 +4224,19 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_scalar(ptr inreg %ptr, i64 inreg %
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB66_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB66_1
@@ -4289,25 +4261,23 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB67_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_or_i64_ret_offset_scalar:
@@ -4323,25 +4293,23 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB67_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_or_i64_ret_offset_scalar:
@@ -4351,19 +4319,19 @@ define amdgpu_gfx i64 @flat_atomic_or_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB67_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_or_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_or_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_or_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_or_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB67_1
@@ -4792,13 +4760,13 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -4824,13 +4792,13 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -4851,13 +4819,13 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB74_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX9-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4888,25 +4856,23 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX7-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB75_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
@@ -4922,25 +4888,23 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX8-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB75_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_xor_i64_noret_offset_scalar:
@@ -4950,13 +4914,13 @@ define amdgpu_gfx void @flat_atomic_xor_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB75_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_xor_b32_e32 v1, s7, v3
; GFX9-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4987,19 +4951,19 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB76_1
@@ -5019,19 +4983,19 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB76_1
@@ -5046,19 +5010,19 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB76_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB76_1
@@ -5083,25 +5047,23 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX7-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX7-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB77_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
@@ -5117,25 +5079,23 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX8-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB77_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_xor_i64_ret_offset_scalar:
@@ -5145,19 +5105,19 @@ define amdgpu_gfx i64 @flat_atomic_xor_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB77_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_xor_b32_e32 v1, s7, v3
-; GFX9-NEXT: v_xor_b32_e32 v0, s6, v2
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_xor_b32_e32 v5, s7, v7
+; GFX9-NEXT: v_xor_b32_e32 v4, s6, v6
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB77_1
@@ -5598,16 +5558,16 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -5633,16 +5593,16 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -5663,16 +5623,16 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -5703,28 +5663,26 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
; GFX7-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB85_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_max_i64_noret_offset_scalar:
@@ -5740,28 +5698,26 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
; GFX8-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB85_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_max_i64_noret_offset_scalar:
@@ -5771,16 +5727,16 @@ define amdgpu_gfx void @flat_atomic_max_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -5811,22 +5767,22 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB86_1
@@ -5846,22 +5802,22 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB86_1
@@ -5876,22 +5832,22 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB86_1
@@ -5916,28 +5872,26 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB87_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_max_i64_ret_offset_scalar:
@@ -5953,28 +5907,26 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB87_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_max_i64_ret_offset_scalar:
@@ -5984,22 +5936,22 @@ define amdgpu_gfx i64 @flat_atomic_max_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB87_1
@@ -6022,28 +5974,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB88_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -6058,28 +6008,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB88_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -6092,28 +6040,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v7, s2
; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB88_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -6136,32 +6082,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB89_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_max_i64_ret_addr64_offset:
@@ -6175,69 +6119,65 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB89_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB89_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -6256,28 +6196,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX7-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX7-NEXT: s_add_u32 s0, s0, s4
; GFX7-NEXT: s_addc_u32 s1, s1, s5
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB90_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -6290,28 +6228,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX8-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX8-NEXT: s_add_u32 s0, s0, s4
; GFX8-NEXT: s_addc_u32 s1, s1, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB90_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -6324,28 +6260,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr %out, i64 %in, i64 %index)
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v7, s2
; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB90_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -6365,32 +6299,30 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB91_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_max_i64_ret_addr64:
@@ -6402,69 +6334,65 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB91_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB91_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -6902,16 +6830,16 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB98_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -6937,16 +6865,16 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB98_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -6967,16 +6895,16 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -7007,28 +6935,26 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
; GFX7-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB99_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
@@ -7044,28 +6970,26 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
; GFX8-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB99_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_umax_i64_noret_offset_scalar:
@@ -7075,16 +6999,16 @@ define amdgpu_gfx void @flat_atomic_umax_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -7115,22 +7039,22 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB100_1
@@ -7150,22 +7074,22 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB100_1
@@ -7180,22 +7104,22 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB100_1
@@ -7220,28 +7144,26 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB101_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
@@ -7257,28 +7179,26 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB101_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_umax_i64_ret_offset_scalar:
@@ -7288,22 +7208,22 @@ define amdgpu_gfx i64 @flat_atomic_umax_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB101_1
@@ -7326,28 +7246,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB102_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -7362,28 +7280,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB102_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -7396,28 +7312,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr %out, i64 %in, i64
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v7, s2
; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB102_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -7440,32 +7354,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB103_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umax_i64_ret_addr64_offset:
@@ -7479,69 +7391,65 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr %out, ptr %out2
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB103_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB103_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -7561,32 +7469,30 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB104_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_umax_i64_ret_addr64:
@@ -7598,69 +7504,65 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr %out, ptr %out2, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB104_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB104_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -8098,16 +8000,16 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -8133,16 +8035,16 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -8163,16 +8065,16 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_scalar(ptr inreg %ptr, i64 in
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -8203,28 +8105,26 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
; GFX7-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB112_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
@@ -8240,28 +8140,26 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
; GFX8-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB112_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_umin_i64_noret_offset_scalar:
@@ -8271,16 +8169,16 @@ define amdgpu_gfx void @flat_atomic_umin_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -8311,22 +8209,22 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB113_1
@@ -8346,22 +8244,22 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB113_1
@@ -8376,22 +8274,22 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB113_1
@@ -8416,28 +8314,26 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB114_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
@@ -8453,28 +8349,26 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB114_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_umin_i64_ret_offset_scalar:
@@ -8484,22 +8378,22 @@ define amdgpu_gfx i64 @flat_atomic_umin_i64_ret_offset_scalar(ptr inreg %out, i6
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB114_1
@@ -8940,16 +8834,16 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -8975,16 +8869,16 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -9005,16 +8899,16 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_scalar(ptr inreg %ptr, i64 inr
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9045,28 +8939,26 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
; GFX7-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB122_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_min_i64_noret_offset_scalar:
@@ -9082,28 +8974,26 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
; GFX8-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB122_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_min_i64_noret_offset_scalar:
@@ -9113,16 +9003,16 @@ define amdgpu_gfx void @flat_atomic_min_i64_noret_offset_scalar(ptr inreg %out,
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s7
+; GFX9-NEXT: v_mov_b32_e32 v7, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9153,22 +9043,22 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB123_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB123_1
@@ -9188,22 +9078,22 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB123_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB123_1
@@ -9218,22 +9108,22 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_scalar(ptr inreg %ptr, i64 inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB123_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB123_1
@@ -9258,28 +9148,26 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB124_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_min_i64_ret_offset_scalar:
@@ -9295,28 +9183,26 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB124_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_min_i64_ret_offset_scalar:
@@ -9326,22 +9212,22 @@ define amdgpu_gfx i64 @flat_atomic_min_i64_ret_offset_scalar(ptr inreg %out, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB124_1
@@ -9364,28 +9250,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX7-NEXT: s_addc_u32 s1, s1, s5
; GFX7-NEXT: s_add_u32 s0, s0, 32
; GFX7-NEXT: s_addc_u32 s1, s1, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
; GFX7-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB125_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX7-NEXT: s_endpgm
@@ -9400,28 +9284,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX8-NEXT: s_addc_u32 s1, s1, s5
; GFX8-NEXT: s_add_u32 s0, s0, 32
; GFX8-NEXT: s_addc_u32 s1, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
; GFX8-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB125_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX8-NEXT: s_endpgm
@@ -9434,28 +9316,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr %out, i64 %in, i64 %
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
-; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[4:5] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v7, s2
; GFX9-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB125_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
@@ -9478,32 +9358,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX7-NEXT: s_addc_u32 s1, s1, 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB126_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_min_i64_ret_addr64_offset:
@@ -9517,69 +9395,65 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr %out, ptr %out2,
; GFX8-NEXT: s_addc_u32 s1, s1, 0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB126_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB126_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -9598,16 +9472,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v6, s3
+; GFX7-NEXT: v_mov_b32_e32 v7, s2
+; GFX7-NEXT: v_mov_b32_e32 v5, s1
+; GFX7-NEXT: v_mov_b32_e32 v4, s0
; GFX7-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v6, s2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
@@ -9628,16 +9502,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v6, s3
+; GFX8-NEXT: v_mov_b32_e32 v7, s2
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
; GFX8-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v6, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
@@ -9658,16 +9532,16 @@ define amdgpu_kernel void @atomic_min_i64(ptr %out, i64 %in) {
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v7, s2
+; GFX9-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s0
; GFX9-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9694,32 +9568,30 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX7-NEXT: s_addc_u32 s1, s1, s7
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX7-NEXT: s_mov_b64 s[6:7], 0
+; GFX7-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s5
+; GFX7-NEXT: v_mov_b32_e32 v5, s4
; GFX7-NEXT: .LBB128_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v0, s5
-; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v9, v3
+; GFX7-NEXT: v_mov_b32_e32 v8, v2
+; GFX7-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX7-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX7-NEXT: s_cbranch_execnz .LBB128_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX7-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: atomic_min_i64_ret_addr64:
@@ -9731,69 +9603,65 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr %out, ptr %out2, i64 %i
; GFX8-NEXT: s_addc_u32 s1, s1, s7
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_mov_b64 s[0:1], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_mov_b32_e32 v5, s4
; GFX8-NEXT: .LBB128_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v9, v3
+; GFX8-NEXT: v_mov_b32_e32 v8, v2
+; GFX8-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX8-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX8-NEXT: s_cbranch_execnz .LBB128_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
+; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s13
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v6, s12
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v9, v3
+; GFX9-NEXT: v_mov_b32_e32 v8, v2
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[8:9]
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB128_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v2, s10
-; GFX9-NEXT: v_mov_b32_e32 v3, s11
-; GFX9-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: v_mov_b32_e32 v1, s11
+; GFX9-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX9-NEXT: s_endpgm
entry:
%ptr = getelementptr inbounds i64, ptr %out, i64 %index
@@ -10255,14 +10123,14 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s4
+; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: .LBB135_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -10290,14 +10158,14 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: .LBB135_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -10320,14 +10188,14 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB135_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -10360,15 +10228,13 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB136_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -10376,12 +10242,12 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB136_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
@@ -10397,15 +10263,13 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB136_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -10413,12 +10277,12 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB136_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_uinc_wrap_i64_noret_offset_scalar:
@@ -10428,14 +10292,14 @@ define amdgpu_gfx void @flat_atomic_uinc_wrap_i64_noret_offset_scalar(ptr inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: .LBB136_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
@@ -10468,22 +10332,22 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[34:35], 0
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB137_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v6
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB137_1
@@ -10503,22 +10367,22 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[34:35], 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB137_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB137_1
@@ -10533,22 +10397,22 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB137_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB137_1
@@ -10573,28 +10437,26 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: s_mov_b64 s[34:35], 0
; GFX7-NEXT: .LBB138_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_mov_b32_e32 v4, s34
-; GFX7-NEXT: v_mov_b32_e32 v5, s35
-; GFX7-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mov_b32_e32 v7, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v6
+; GFX7-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX7-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX7-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX7-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_cbranch_execnz .LBB138_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX7-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
@@ -10610,28 +10472,26 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: s_mov_b64 s[34:35], 0
; GFX8-NEXT: .LBB138_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s34
-; GFX8-NEXT: v_mov_b32_e32 v5, s35
-; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_mov_b32_e32 v7, v1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; GFX8-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; GFX8-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_cbranch_execnz .LBB138_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
+; GFX8-NEXT: s_or_b64 exec, exec, s[34:35]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_uinc_wrap_i64_ret_offset_scalar:
@@ -10641,22 +10501,22 @@ define amdgpu_gfx i64 @flat_atomic_uinc_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB138_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, 1, v6
+; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc
+; GFX9-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; GFX9-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB138_1
@@ -11133,17 +10993,17 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX7-NEXT: flat_load_dword v2, v[0:1]
; GFX7-NEXT: flat_load_dword v3, v[3:4]
; GFX7-NEXT: s_mov_b64 s[36:37], 0
-; GFX7-NEXT: .LBB145_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
; GFX7-NEXT: v_mov_b32_e32 v6, s7
; GFX7-NEXT: v_mov_b32_e32 v7, s6
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX7-NEXT: v_mov_b32_e32 v4, s4
; GFX7-NEXT: v_mov_b32_e32 v5, s5
+; GFX7-NEXT: .LBB145_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX7-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11171,17 +11031,17 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX8-NEXT: flat_load_dword v2, v[0:1]
; GFX8-NEXT: flat_load_dword v3, v[3:4]
; GFX8-NEXT: s_mov_b64 s[36:37], 0
-; GFX8-NEXT: .LBB145_1: ; %atomicrmw.start
-; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: v_mov_b32_e32 v6, s7
; GFX8-NEXT: v_mov_b32_e32 v7, s6
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: v_mov_b32_e32 v5, s5
+; GFX8-NEXT: .LBB145_1: ; %atomicrmw.start
+; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX8-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11204,17 +11064,17 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_scalar(ptr inreg %ptr, i
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
-; GFX9-NEXT: .LBB145_1: ; %atomicrmw.start
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX9-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-NEXT: v_mov_b32_e32 v6, s7
; GFX9-NEXT: v_mov_b32_e32 v7, s6
-; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v5, s5
+; GFX9-NEXT: .LBB145_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11237,28 +11097,26 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GFX7-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_add_u32 s36, s4, 32
-; GFX7-NEXT: s_addc_u32 s37, s5, 0
-; GFX7-NEXT: s_add_u32 s34, s4, 36
+; GFX7-NEXT: s_add_u32 s34, s4, 32
; GFX7-NEXT: s_addc_u32 s35, s5, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s34
-; GFX7-NEXT: v_mov_b32_e32 v1, s35
-; GFX7-NEXT: v_mov_b32_e32 v4, s36
-; GFX7-NEXT: v_mov_b32_e32 v5, s37
+; GFX7-NEXT: s_add_u32 s36, s4, 36
+; GFX7-NEXT: s_addc_u32 s37, s5, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s36
+; GFX7-NEXT: v_mov_b32_e32 v1, s37
+; GFX7-NEXT: v_mov_b32_e32 v4, s34
+; GFX7-NEXT: v_mov_b32_e32 v5, s35
; GFX7-NEXT: flat_load_dword v3, v[0:1]
; GFX7-NEXT: flat_load_dword v2, v[4:5]
-; GFX7-NEXT: s_mov_b64 s[38:39], 0
+; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s7
+; GFX7-NEXT: v_mov_b32_e32 v7, s6
; GFX7-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; GFX7-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX7-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s7
-; GFX7-NEXT: v_mov_b32_e32 v7, s6
-; GFX7-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GFX7-NEXT: v_mov_b32_e32 v4, s36
-; GFX7-NEXT: v_mov_b32_e32 v5, s37
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v2
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX7-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11266,39 +11124,37 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GFX7-NEXT: buffer_wbinvl1_vol
; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
+; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX7-NEXT: s_cbranch_execnz .LBB146_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]
+; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s36, s4, 32
-; GFX8-NEXT: s_addc_u32 s37, s5, 0
-; GFX8-NEXT: s_add_u32 s34, s4, 36
+; GFX8-NEXT: s_add_u32 s34, s4, 32
; GFX8-NEXT: s_addc_u32 s35, s5, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s34
-; GFX8-NEXT: v_mov_b32_e32 v1, s35
-; GFX8-NEXT: v_mov_b32_e32 v4, s36
-; GFX8-NEXT: v_mov_b32_e32 v5, s37
+; GFX8-NEXT: s_add_u32 s36, s4, 36
+; GFX8-NEXT: s_addc_u32 s37, s5, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s36
+; GFX8-NEXT: v_mov_b32_e32 v1, s37
+; GFX8-NEXT: v_mov_b32_e32 v4, s34
+; GFX8-NEXT: v_mov_b32_e32 v5, s35
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: flat_load_dword v2, v[4:5]
-; GFX8-NEXT: s_mov_b64 s[38:39], 0
+; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: v_mov_b32_e32 v6, s7
+; GFX8-NEXT: v_mov_b32_e32 v7, s6
; GFX8-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; GFX8-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX8-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: v_mov_b32_e32 v7, s6
-; GFX8-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GFX8-NEXT: v_mov_b32_e32 v4, s36
-; GFX8-NEXT: v_mov_b32_e32 v5, s37
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX8-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11306,12 +11162,12 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GFX8-NEXT: buffer_wbinvl1_vol
; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
+; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX8-NEXT: s_cbranch_execnz .LBB146_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]
+; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_udec_wrap_i64_noret_offset_scalar:
@@ -11321,17 +11177,17 @@ define amdgpu_gfx void @flat_atomic_udec_wrap_i64_noret_offset_scalar(ptr inreg
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[2:3], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[36:37], 0
-; GFX9-NEXT: .LBB146_1: ; %atomicrmw.start
-; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX9-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
; GFX9-NEXT: v_mov_b32_e32 v6, s7
; GFX9-NEXT: v_mov_b32_e32 v7, s6
-; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_mov_b32_e32 v5, s5
+; GFX9-NEXT: .LBB146_1: ; %atomicrmw.start
+; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
@@ -11364,25 +11220,25 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX7-NEXT: flat_load_dword v0, v[0:1]
; GFX7-NEXT: flat_load_dword v1, v[2:3]
; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[34:35], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX7-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_mov_b32_e32 v5, s5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX7-NEXT: s_cbranch_execnz .LBB147_1
@@ -11402,25 +11258,25 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX8-NEXT: flat_load_dword v0, v[0:1]
; GFX8-NEXT: flat_load_dword v1, v[2:3]
; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX8-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX8-NEXT: s_cbranch_execnz .LBB147_1
@@ -11435,25 +11291,25 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_scalar(ptr inreg %ptr, i64
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_subrev_co_u32_e64 v7, s[34:35], 1, v2
-; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v8
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB147_1
@@ -11468,81 +11324,77 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GFX7-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_add_u32 s36, s4, 32
-; GFX7-NEXT: s_addc_u32 s37, s5, 0
-; GFX7-NEXT: s_add_u32 s34, s4, 36
+; GFX7-NEXT: s_add_u32 s34, s4, 32
; GFX7-NEXT: s_addc_u32 s35, s5, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s34
-; GFX7-NEXT: v_mov_b32_e32 v1, s35
-; GFX7-NEXT: v_mov_b32_e32 v2, s36
-; GFX7-NEXT: v_mov_b32_e32 v3, s37
+; GFX7-NEXT: s_add_u32 s36, s4, 36
+; GFX7-NEXT: s_addc_u32 s37, s5, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s36
+; GFX7-NEXT: v_mov_b32_e32 v1, s37
+; GFX7-NEXT: v_mov_b32_e32 v2, s34
+; GFX7-NEXT: v_mov_b32_e32 v3, s35
; GFX7-NEXT: flat_load_dword v1, v[0:1]
; GFX7-NEXT: flat_load_dword v0, v[2:3]
-; GFX7-NEXT: s_mov_b64 s[38:39], 0
+; GFX7-NEXT: s_mov_b64 s[36:37], 0
+; GFX7-NEXT: v_mov_b32_e32 v4, s7
+; GFX7-NEXT: v_mov_b32_e32 v5, s6
; GFX7-NEXT: .LBB148_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v2, v0
-; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX7-NEXT: v_subrev_i32_e64 v7, s[34:35], 1, v2
-; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX7-NEXT: v_mov_b32_e32 v0, s7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
+; GFX7-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX7-NEXT: v_subrev_i32_e64 v0, s[34:35], 1, v8
+; GFX7-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX7-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s36
-; GFX7-NEXT: v_mov_b32_e32 v5, s37
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX7-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX7-NEXT: buffer_wbinvl1_vol
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX7-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX7-NEXT: s_cbranch_execnz .LBB148_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[38:39]
+; GFX7-NEXT: s_or_b64 exec, exec, s[36:37]
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s36, s4, 32
-; GFX8-NEXT: s_addc_u32 s37, s5, 0
-; GFX8-NEXT: s_add_u32 s34, s4, 36
+; GFX8-NEXT: s_add_u32 s34, s4, 32
; GFX8-NEXT: s_addc_u32 s35, s5, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s34
-; GFX8-NEXT: v_mov_b32_e32 v1, s35
-; GFX8-NEXT: v_mov_b32_e32 v2, s36
-; GFX8-NEXT: v_mov_b32_e32 v3, s37
+; GFX8-NEXT: s_add_u32 s36, s4, 36
+; GFX8-NEXT: s_addc_u32 s37, s5, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s36
+; GFX8-NEXT: v_mov_b32_e32 v1, s37
+; GFX8-NEXT: v_mov_b32_e32 v2, s34
+; GFX8-NEXT: v_mov_b32_e32 v3, s35
; GFX8-NEXT: flat_load_dword v1, v[0:1]
; GFX8-NEXT: flat_load_dword v0, v[2:3]
-; GFX8-NEXT: s_mov_b64 s[38:39], 0
+; GFX8-NEXT: s_mov_b64 s[36:37], 0
+; GFX8-NEXT: v_mov_b32_e32 v4, s7
+; GFX8-NEXT: v_mov_b32_e32 v5, s6
; GFX8-NEXT: .LBB148_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, v0
-; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX8-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX8-NEXT: v_mov_b32_e32 v0, s7
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
+; GFX8-NEXT: v_mov_b32_e32 v9, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, v0
+; GFX8-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX8-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; GFX8-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX8-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s36
-; GFX8-NEXT: v_mov_b32_e32 v5, s37
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX8-NEXT: buffer_wbinvl1_vol
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX8-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; GFX8-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX8-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; GFX8-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX8-NEXT: s_cbranch_execnz .LBB148_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX8-NEXT: s_or_b64 exec, exec, s[38:39]
+; GFX8-NEXT: s_or_b64 exec, exec, s[36:37]
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: flat_atomic_udec_wrap_i64_ret_offset_scalar:
@@ -11552,25 +11404,25 @@ define amdgpu_gfx i64 @flat_atomic_udec_wrap_i64_ret_offset_scalar(ptr inreg %ou
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: flat_load_dwordx2 v[0:1], v[0:1] offset:32
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s7
+; GFX9-NEXT: v_mov_b32_e32 v5, s6
+; GFX9-NEXT: v_mov_b32_e32 v2, s4
+; GFX9-NEXT: v_mov_b32_e32 v3, s5
; GFX9-NEXT: .LBB148_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v1
-; GFX9-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_subrev_co_u32_e64 v7, s[34:35], 1, v2
-; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
+; GFX9-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v8
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_mov_b32_e32 v5, s5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] offset:32 glc
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; GFX9-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB148_1
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index deaa6a73d0bf4..ed9ad86926103 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -2196,23 +2196,23 @@ main_body:
define amdgpu_kernel void @flat_atomic_fadd_f64_noret_pat_agent_safe(ptr %ptr) {
; GFX90A-LABEL: flat_atomic_fadd_f64_noret_pat_agent_safe:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-NEXT: s_mov_b64 s[2:3], 0
+; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX90A-NEXT: s_mov_b64 s[0:1], 0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[0:1], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[2:3], s[2:3] op_sel:[0,1]
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: v_add_f64 v[0:1], v[2:3], 4.0
-; GFX90A-NEXT: v_pk_mov_b32 v[4:5], s[0:1], s[0:1] op_sel:[0,1]
; GFX90A-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; GFX90A-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1_vol
; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; GFX90A-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX90A-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], v[0:1], v[0:1] op_sel:[0,1]
-; GFX90A-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX90A-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 4134420e78a2e..112cfabd6d434 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -3696,8 +3696,8 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1
; SI-NEXT: s_sub_i32 s0, s7, s13
; SI-NEXT: s_add_i32 s12, s0, 26
; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_mov_b32_e32 v6, 0
; SI-NEXT: v_mov_b32_e32 v10, 0x43300000
+; SI-NEXT: v_mov_b32_e32 v6, 0
; SI-NEXT: .LBB6_6: ; %frem.loop_body
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_mov_b32_e32 v9, v3
@@ -16516,8 +16516,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: s_sub_i32 s0, s13, s17
; SI-NEXT: s_add_i32 s16, s0, 26
; SI-NEXT: s_mov_b32 s13, 0x432fffff
-; SI-NEXT: v_mov_b32_e32 v6, 0
; SI-NEXT: v_mov_b32_e32 v10, 0x43300000
+; SI-NEXT: v_mov_b32_e32 v6, 0
; SI-NEXT: .LBB13_6: ; %frem.loop_body23
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_mov_b32_e32 v9, v3
@@ -16643,8 +16643,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i
; SI-NEXT: s_sub_i32 s0, s13, s17
; SI-NEXT: s_add_i32 s16, s0, 26
; SI-NEXT: s_mov_b32 s13, 0x432fffff
-; SI-NEXT: v_mov_b32_e32 v8, 0
; SI-NEXT: v_mov_b32_e32 v12, 0x43300000
+; SI-NEXT: v_mov_b32_e32 v8, 0
; SI-NEXT: .LBB13_16: ; %frem.loop_body
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_mov_b32_e32 v11, v5
@@ -18334,9 +18334,9 @@ define amdgpu_kernel void @frem_v2f64_const_one_denum(ptr addrspace(1) %out, ptr
; SI-NEXT: s_cbranch_scc1 .LBB15_8
; SI-NEXT: ; %bb.5: ; %frem.loop_body23.preheader
; SI-NEXT: s_add_i32 s9, s7, 25
+; SI-NEXT: v_mov_b32_e32 v5, 0x43300000
; SI-NEXT: v_mov_b32_e32 v0, 0
; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_mov_b32_e32 v5, 0x43300000
; SI-NEXT: .LBB15_6: ; %frem.loop_body23
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_mov_b32_e32 v4, v2
@@ -18424,9 +18424,9 @@ define amdgpu_kernel void @frem_v2f64_const_one_denum(ptr addrspace(1) %out, ptr
; SI-NEXT: s_cbranch_scc1 .LBB15_18
; SI-NEXT: ; %bb.15: ; %frem.loop_body.preheader
; SI-NEXT: s_add_i32 s9, s7, 25
+; SI-NEXT: v_mov_b32_e32 v7, 0x43300000
; SI-NEXT: v_mov_b32_e32 v2, 0
; SI-NEXT: s_mov_b32 s7, 0x432fffff
-; SI-NEXT: v_mov_b32_e32 v7, 0x43300000
; SI-NEXT: .LBB15_16: ; %frem.loop_body
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_mov_b32_e32 v6, v4
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index f1fc6066b1e5f..c1089c70368be 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -21465,43 +21465,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -21516,39 +21517,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -21563,34 +21566,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB78_1
@@ -21602,41 +21605,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB78_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -21644,33 +21647,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB78_1
@@ -21684,34 +21687,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB78_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB78_1
@@ -21864,43 +21867,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -21915,39 +21919,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -21962,34 +21968,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB79_1
@@ -22001,41 +22007,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB79_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -22043,33 +22049,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB79_1
@@ -22085,34 +22091,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB79_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB79_1
@@ -22265,43 +22271,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -22316,39 +22323,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -22363,34 +22372,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB80_1
@@ -22402,41 +22411,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB80_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -22444,33 +22453,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB80_1
@@ -22486,34 +22495,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__offset12b_neg__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB80_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB80_1
@@ -22673,44 +22682,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -22723,41 +22732,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -22770,35 +22779,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB81_1
@@ -22809,36 +22818,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB81_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -22848,36 +22857,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
+; GFX908-NEXT: s_waitcnt vmcnt(0)
+; GFX908-NEXT: buffer_wbinvl1
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB81_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -22887,37 +22896,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX8-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB81_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB81_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23059,44 +23068,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -23109,41 +23118,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -23156,35 +23165,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB82_1
@@ -23195,36 +23204,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB82_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23234,36 +23243,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB82_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23275,37 +23284,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB82_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB82_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23448,44 +23457,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -23498,41 +23507,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -23545,35 +23554,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB83_1
@@ -23584,36 +23593,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB83_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23623,36 +23632,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB83_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23664,37 +23673,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB83_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB83_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -23847,43 +23856,44 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -23898,39 +23908,41 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -23945,34 +23957,34 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB84_1
@@ -23984,44 +23996,44 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX90A-LABEL: global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB84_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -24029,33 +24041,33 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB84_1
@@ -24071,34 +24083,34 @@ define <2 x bfloat> @global_system_atomic_fadd_ret_v2bf16__offset12b_pos__amdgpu
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB84_1
@@ -24251,44 +24263,44 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -24301,41 +24313,41 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-FAKE16-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -24348,35 +24360,35 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX10-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB85_1
@@ -24387,39 +24399,39 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX90A-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB85_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -24429,36 +24441,36 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX908-LABEL: global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB85_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -24470,37 +24482,37 @@ define void @global_system_atomic_fadd_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB85_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -24644,43 +24656,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -24695,39 +24708,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -24742,34 +24757,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB86_1
@@ -24781,41 +24796,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB86_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memory:
@@ -24823,33 +24838,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB86_1
@@ -24863,34 +24878,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_remote_memor
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB86_1
@@ -25042,44 +25057,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -25092,41 +25107,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -25139,35 +25154,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB87_1
@@ -25178,36 +25193,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB87_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -25217,36 +25232,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB87_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -25256,37 +25271,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory(ptr
; GFX8-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB87_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -25429,43 +25444,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -25480,39 +25496,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -25527,34 +25545,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB88_1
@@ -25566,41 +25584,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB88_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -25608,33 +25626,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB88_1
@@ -25648,34 +25666,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__amdgpu_no_fine_grained
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB88_1
@@ -25827,44 +25845,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -25877,41 +25895,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -25924,35 +25942,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB89_1
@@ -25963,36 +25981,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB89_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26002,36 +26020,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB89_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26041,37 +26059,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX8-LABEL: global_agent_atomic_fadd_noret_v2bf16__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB89_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26214,43 +26232,44 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -26265,39 +26284,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -26312,34 +26333,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB90_1
@@ -26351,41 +26372,41 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX90A-LABEL: global_agent_atomic_fadd_ret_v2bf16__maybe_remote:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB90_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fadd_ret_v2bf16__maybe_remote:
@@ -26393,33 +26414,33 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB90_1
@@ -26433,34 +26454,34 @@ define <2 x bfloat> @global_agent_atomic_fadd_ret_v2bf16__maybe_remote(ptr addrs
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_add_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB90_1
@@ -26612,44 +26633,44 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -26662,41 +26683,41 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -26709,35 +26730,35 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX10-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB91_1
@@ -26748,36 +26769,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX90A-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB91_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26787,36 +26808,36 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX908-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB91_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -26826,37 +26847,37 @@ define void @global_agent_atomic_fadd_noret_v2bf16__maybe_remote(ptr addrspace(1
; GFX8-LABEL: global_agent_atomic_fadd_noret_v2bf16__maybe_remote:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_add_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB91_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 4b0e5cd77710e..2b5feeb686731 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -32,27 +32,27 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB0_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -78,25 +78,25 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory:
@@ -104,13 +104,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -194,27 +194,27 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -240,25 +240,25 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -266,13 +266,13 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -358,27 +358,27 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -404,25 +404,25 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -430,13 +430,13 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -522,22 +522,22 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB3_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -567,20 +567,20 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -590,20 +590,20 @@ define void @global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -678,22 +678,22 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -723,20 +723,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -746,20 +746,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -837,22 +837,22 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -882,20 +882,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -905,20 +905,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -997,27 +997,27 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB6_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1043,28 +1043,28 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1072,13 +1072,13 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1165,22 +1165,22 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1210,23 +1210,23 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1236,20 +1236,20 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1327,41 +1327,42 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_max_f32 v3, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX11-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -1381,14 +1382,14 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -1406,25 +1407,25 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory:
@@ -1432,13 +1433,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1557,27 +1558,27 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1603,25 +1604,25 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1629,13 +1630,13 @@ define float @global_agent_atomic_fmax_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1723,27 +1724,27 @@ define float @global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1769,25 +1770,25 @@ define float @global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1795,13 +1796,13 @@ define float @global_agent_atomic_fmax_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1885,27 +1886,27 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1931,25 +1932,25 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1957,13 +1958,13 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2049,27 +2050,27 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX942-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
@@ -2095,25 +2096,25 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
@@ -2121,13 +2122,13 @@ define float @global_agent_atomic_fmax_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2213,22 +2214,22 @@ define void @global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2258,20 +2259,20 @@ define void @global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2281,20 +2282,20 @@ define void @global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2369,22 +2370,22 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB14_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2414,20 +2415,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2437,20 +2438,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2528,22 +2529,22 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX942-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2573,20 +2574,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2596,20 +2597,20 @@ define void @global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX908-LABEL: global_agent_atomic_fmax_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2688,27 +2689,27 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX942-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB16_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2734,28 +2735,28 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX90A-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2763,13 +2764,13 @@ define float @global_system_atomic_fmax_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2856,22 +2857,22 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX942-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2901,23 +2902,23 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX90A-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2927,20 +2928,20 @@ define void @global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX908-LABEL: global_system_atomic_fmax_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3014,15 +3015,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3052,15 +3053,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3098,15 +3099,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3124,15 +3125,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3187,15 +3188,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3225,15 +3226,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3271,15 +3272,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3299,15 +3300,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3361,15 +3362,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3399,15 +3400,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3445,15 +3446,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3473,15 +3474,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_max_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3534,21 +3535,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3571,22 +3572,22 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3616,21 +3617,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3640,21 +3641,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3697,21 +3698,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3734,22 +3735,22 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3779,21 +3780,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3805,21 +3806,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3863,21 +3864,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3900,22 +3901,22 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX11-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3945,21 +3946,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3971,21 +3972,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4030,15 +4031,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4068,15 +4069,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -4096,15 +4097,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v6, v4
-; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX10-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX10-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX10-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX10-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -4123,41 +4124,41 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX90A-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
-; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX90A-NEXT: v_max_f64 v[2:3], v[2:3], v[6:7]
+; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v4
-; GFX90A-NEXT: v_mov_b32_e32 v1, v5
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -4175,15 +4176,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4201,32 +4202,30 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_mov_b32_e32 v7, v1
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
+; GFX7-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[10:11], v[10:11]
-; GFX7-NEXT: v_max_f64 v[8:9], v[2:3], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v2, v10
-; GFX7-NEXT: v_mov_b32_e32 v0, v8
-; GFX7-NEXT: v_mov_b32_e32 v1, v9
-; GFX7-NEXT: v_mov_b32_e32 v3, v11
-; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX7-NEXT: v_max_f64 v[6:7], v[0:1], v[10:11]
+; GFX7-NEXT: v_mov_b32_e32 v2, v8
+; GFX7-NEXT: v_mov_b32_e32 v0, v6
+; GFX7-NEXT: v_mov_b32_e32 v1, v7
+; GFX7-NEXT: v_mov_b32_e32 v3, v9
+; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; GFX7-NEXT: v_mov_b32_e32 v11, v1
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v10, v0
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4237,33 +4236,30 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: v_mov_b32_e32 v7, v1
-; GFX6-NEXT: v_mov_b32_e32 v6, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
+; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
+; GFX6-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_max_f64 v[2:3], v[10:11], v[10:11]
-; GFX6-NEXT: v_max_f64 v[8:9], v[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, v10
-; GFX6-NEXT: v_mov_b32_e32 v0, v8
-; GFX6-NEXT: v_mov_b32_e32 v1, v9
-; GFX6-NEXT: v_mov_b32_e32 v3, v11
-; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX6-NEXT: v_max_f64 v[6:7], v[0:1], v[10:11]
+; GFX6-NEXT: v_mov_b32_e32 v2, v8
+; GFX6-NEXT: v_mov_b32_e32 v0, v6
+; GFX6-NEXT: v_mov_b32_e32 v1, v7
+; GFX6-NEXT: v_mov_b32_e32 v3, v9
+; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; GFX6-NEXT: v_mov_b32_e32 v11, v1
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v10, v0
+; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4283,15 +4279,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4321,15 +4317,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_max_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -4367,15 +4363,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -4393,15 +4389,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_max_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4460,8 +4456,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4473,12 +4470,11 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4506,8 +4502,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4519,12 +4516,11 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4556,13 +4552,13 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -4584,8 +4580,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4597,12 +4594,11 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4626,8 +4622,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4639,12 +4636,11 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4668,6 +4664,7 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -4679,10 +4676,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4711,13 +4707,13 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
@@ -4745,14 +4741,14 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -4779,17 +4775,17 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4896,33 +4892,35 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -4933,7 +4931,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4946,24 +4944,25 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4990,67 +4989,69 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -5061,18 +5062,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5082,12 +5084,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5112,9 +5113,10 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -5123,10 +5125,9 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5146,36 +5147,36 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5191,14 +5192,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -5226,17 +5227,17 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -5347,33 +5348,35 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -5384,7 +5387,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5397,24 +5400,25 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5442,67 +5446,69 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -5513,18 +5519,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5534,12 +5541,11 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5564,9 +5570,10 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -5575,10 +5582,9 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5598,36 +5604,36 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5643,14 +5649,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -5678,17 +5684,17 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -5800,8 +5806,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5813,10 +5820,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5845,8 +5851,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5858,10 +5865,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5894,13 +5900,13 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_max_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -5921,8 +5927,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5934,10 +5941,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5962,8 +5968,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5975,10 +5982,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -6003,6 +6009,7 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -6014,9 +6021,8 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX10-NEXT: v_max_f16_e32 v3, v3, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6045,13 +6051,13 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_max_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
@@ -6078,13 +6084,13 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
@@ -6111,16 +6117,16 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
+; GFX8-NEXT: v_max_f16_e32 v3, v3, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6226,36 +6232,38 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6272,36 +6280,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6318,30 +6327,30 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB30_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6351,37 +6360,39 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6393,37 +6404,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6435,31 +6446,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB30_1
@@ -6470,31 +6481,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6504,31 +6515,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6538,32 +6549,32 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6666,36 +6677,38 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6712,36 +6725,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6759,30 +6773,30 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB31_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6792,37 +6806,39 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6834,37 +6850,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6876,31 +6892,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB31_1
@@ -6911,31 +6927,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6945,31 +6961,31 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6979,32 +6995,32 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX8-LABEL: global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7108,15 +7124,15 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7143,15 +7159,15 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7173,44 +7189,44 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB32_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7233,15 +7249,15 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7264,14 +7280,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_max_f16_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX10-NEXT: v_max_f16_e32 v3, v3, v2
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7291,27 +7307,27 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -7319,14 +7335,14 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v4, v4
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v5
+; GFX908-NEXT: v_max_f16_e32 v3, v3, v2
; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -7347,19 +7363,19 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
+; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v5, v2, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
@@ -7448,14 +7464,14 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7481,24 +7497,24 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7511,24 +7527,24 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX942-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB33_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7539,14 +7555,14 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7568,25 +7584,25 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7598,23 +7614,23 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX10-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2046
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
+; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB33_1
@@ -7625,22 +7641,22 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX90A-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_max_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7650,22 +7666,22 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX908-LABEL: global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2046
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f16_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX908-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX908-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7677,22 +7693,22 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_max_f16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7777,34 +7793,36 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -7815,7 +7833,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7828,24 +7846,25 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7873,67 +7892,69 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB34_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -7944,18 +7965,19 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7965,12 +7987,11 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7995,9 +8016,10 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -8006,10 +8028,9 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_max_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_max_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -8029,39 +8050,39 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8077,14 +8098,14 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_max_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -8112,17 +8133,17 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -8233,37 +8254,39 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8280,37 +8303,38 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8327,30 +8351,30 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB35_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8360,37 +8384,39 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8402,37 +8428,37 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-FAKE16-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8444,31 +8470,31 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX10-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB35_1
@@ -8479,34 +8505,34 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8515,32 +8541,32 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
;
; GFX908-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8550,32 +8576,32 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX8-LABEL: global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12634,15 +12660,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12661,43 +12687,43 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB46_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -12717,14 +12743,14 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -12742,25 +12768,25 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12768,13 +12794,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -12792,21 +12818,21 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_max_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
@@ -12916,15 +12942,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12943,43 +12969,43 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB47_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -12999,14 +13025,14 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -13024,25 +13050,25 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13050,13 +13076,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13076,21 +13102,21 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
@@ -13200,15 +13226,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13227,43 +13253,43 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB48_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -13283,14 +13309,14 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -13308,25 +13334,25 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13334,13 +13360,13 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13360,21 +13386,21 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB48_1
@@ -13491,21 +13517,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13518,23 +13544,23 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB49_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13544,22 +13570,22 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13571,21 +13597,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB49_1
@@ -13596,20 +13622,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13619,20 +13645,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13642,24 +13668,24 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX8-LABEL: global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13760,21 +13786,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13787,23 +13813,23 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13813,22 +13839,22 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13840,21 +13866,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -13865,20 +13891,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13888,20 +13914,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13913,24 +13939,24 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14032,21 +14058,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14059,23 +14085,23 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14085,22 +14111,22 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX11-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14112,21 +14138,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -14137,20 +14163,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14160,20 +14186,20 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14185,24 +14211,24 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14313,15 +14339,15 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14341,43 +14367,43 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX942-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -14397,14 +14423,14 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -14422,28 +14448,28 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX90A-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14451,13 +14477,13 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14477,21 +14503,21 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -14600,22 +14626,22 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14628,23 +14654,23 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX942-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14654,22 +14680,22 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX11-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14681,21 +14707,21 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX10-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_max_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -14706,23 +14732,23 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX90A-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_max_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14732,20 +14758,20 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX908-LABEL: global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14757,24 +14783,24 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14881,41 +14907,43 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14934,38 +14962,39 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14979,87 +15008,88 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15074,39 +15104,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15121,34 +15153,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -15160,41 +15192,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15202,33 +15234,33 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
@@ -15242,34 +15274,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
@@ -15390,41 +15422,43 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15443,38 +15477,39 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15488,87 +15523,88 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15583,39 +15619,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15630,34 +15668,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -15669,41 +15707,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15711,33 +15749,33 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB55_1
@@ -15753,34 +15791,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
@@ -15901,41 +15939,43 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15954,38 +15994,39 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15999,87 +16040,88 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX942-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16094,39 +16136,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16141,34 +16185,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -16180,41 +16224,41 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16222,33 +16266,33 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -16264,34 +16308,34 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -16419,43 +16463,42 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16472,40 +16515,40 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16518,39 +16561,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16560,44 +16603,44 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16610,41 +16653,41 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16657,35 +16700,35 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -16696,36 +16739,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16735,36 +16778,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16774,37 +16817,37 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX8-LABEL: global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16914,43 +16957,42 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16967,40 +17009,40 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17013,39 +17055,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB58_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17055,44 +17097,44 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17105,41 +17147,41 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17152,35 +17194,35 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB58_1
@@ -17191,36 +17233,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17230,36 +17272,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17271,37 +17313,37 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17412,43 +17454,42 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17465,40 +17506,40 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17511,39 +17552,39 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX942-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB59_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17553,44 +17594,44 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17603,41 +17644,41 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17650,35 +17691,35 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB59_1
@@ -17689,36 +17730,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17728,36 +17769,36 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17769,37 +17810,37 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17919,42 +17960,44 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17973,39 +18016,40 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18019,87 +18063,88 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX942-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB60_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18114,39 +18159,41 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18161,34 +18208,34 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB60_1
@@ -18200,44 +18247,44 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX90A-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18245,33 +18292,33 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_max_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_max_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB60_1
@@ -18287,34 +18334,34 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_max_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_max_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB60_1
@@ -18434,44 +18481,43 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18488,41 +18534,41 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18535,39 +18581,39 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX942-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB61_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18577,44 +18623,44 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18627,41 +18673,41 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-FAKE16-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18674,35 +18720,35 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX10-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB61_1
@@ -18713,39 +18759,39 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX90A-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18755,36 +18801,36 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX908-LABEL: global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18796,37 +18842,37 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_max_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 92bcb54d32d79..1510e0efeeee2 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -32,27 +32,27 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB0_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -78,25 +78,25 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB0_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory:
@@ -104,13 +104,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory(pt
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB0_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -194,27 +194,27 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB1_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -240,25 +240,25 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB1_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -266,13 +266,13 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_gr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -358,27 +358,27 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB2_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -404,25 +404,25 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB2_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -430,13 +430,13 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__amdgpu_no_fine_gr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -522,22 +522,22 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB3_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -567,20 +567,20 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB3_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -590,20 +590,20 @@ define void @global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB3_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -678,22 +678,22 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB4_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -723,20 +723,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB4_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -746,20 +746,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB4_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -837,22 +837,22 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -882,20 +882,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -905,20 +905,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -997,27 +997,27 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX942-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB6_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1043,28 +1043,28 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB6_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -1072,13 +1072,13 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__amdgpu_no_fine_g
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB6_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1165,22 +1165,22 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX942-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB7_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1210,23 +1210,23 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB7_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1236,20 +1236,20 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_
; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB7_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB7_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1327,41 +1327,42 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB8_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_max_f32_e32 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_max_f32 v3, v2, v2
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX11-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -1381,14 +1382,14 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f32_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f32_e32 v3, v4, v4
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -1406,25 +1407,25 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB8_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory:
@@ -1432,13 +1433,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_remote_memory(ptr addr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB8_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1557,27 +1558,27 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB9_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1603,25 +1604,25 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB9_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -1629,13 +1630,13 @@ define float @global_agent_atomic_fmin_ret_f32__amdgpu_no_fine_grained_memory__a
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB9_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1723,27 +1724,27 @@ define float @global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB10_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1769,25 +1770,25 @@ define float @global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB10_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memory:
@@ -1795,13 +1796,13 @@ define float @global_agent_atomic_fmin_ret_f32__ftz__amdgpu_no_fine_grained_memo
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -1885,27 +1886,27 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB11_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1931,25 +1932,25 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB11_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -1957,13 +1958,13 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2049,27 +2050,27 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX942-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB12_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
@@ -2095,25 +2096,25 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB12_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
@@ -2121,13 +2122,13 @@ define float @global_agent_atomic_fmin_ret_f32__offset12b_neg__ftz__amdgpu_no_fi
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB12_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2213,22 +2214,22 @@ define void @global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB13_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2258,20 +2259,20 @@ define void @global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB13_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2281,20 +2282,20 @@ define void @global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_mem
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB13_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB13_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2369,22 +2370,22 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB14_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2414,20 +2415,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB14_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2437,20 +2438,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB14_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB14_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2528,22 +2529,22 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX942-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB15_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2573,20 +2574,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB15_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2596,20 +2597,20 @@ define void @global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_f
; GFX908-LABEL: global_agent_atomic_fmin_noret_f32__offset12b_neg__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB15_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB15_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2688,27 +2689,27 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX942-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB16_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2734,28 +2735,28 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX90A-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB16_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
@@ -2763,13 +2764,13 @@ define float @global_system_atomic_fmin_ret_f32__offset12b_pos__ftz__amdgpu_no_f
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v2
; GFX908-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f32_e32 v5, v2, v2
; GFX908-NEXT: v_max_f32_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -2856,22 +2857,22 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX942-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f32_e32 v4, v2, v2
; GFX942-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f32_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX942-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB17_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2901,23 +2902,23 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX90A-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f32_e32 v4, v2, v2
; GFX90A-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f32_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f32_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f32_e32 v4, v4, v3
+; GFX90A-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB17_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2927,20 +2928,20 @@ define void @global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_
; GFX908-LABEL: global_system_atomic_fmin_noret_f32__offset12b_pos__ftz__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f32_e32 v4, v2, v2
; GFX908-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f32_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f32_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_max_f32_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB17_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3014,15 +3015,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3052,15 +3053,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3098,15 +3099,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3124,15 +3125,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3187,15 +3188,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3225,15 +3226,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3271,15 +3272,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3299,15 +3300,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7f8, v0
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3361,15 +3362,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3399,15 +3400,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3445,15 +3446,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -3473,15 +3474,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v5, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v9, v1
; GFX8-NEXT: v_mov_b32_e32 v8, v0
-; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
-; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[6:7]
+; GFX8-NEXT: v_min_f64 v[6:7], v[0:1], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[6:9] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -3534,21 +3535,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3571,22 +3572,22 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3616,21 +3617,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3640,21 +3641,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX8-LABEL: global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB21_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3697,21 +3698,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3734,22 +3735,22 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3779,21 +3780,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:2040
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:2040
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:2040 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:2040 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB22_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3805,21 +3806,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7f8, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3863,21 +3864,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3900,22 +3901,22 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX11-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX11-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
+; GFX11-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX11-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -3945,21 +3946,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dwordx2 v[6:7], v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off offset:-2048
+; GFX908-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off offset:-2048 glc
+; GFX908-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX908-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX908-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v7, v5
+; GFX908-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX908-NEXT: v_mov_b32_e32 v5, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v6, v4
+; GFX908-NEXT: v_mov_b32_e32 v4, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB23_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3971,21 +3972,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
+; GFX8-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX8-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX8-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v7, v5
+; GFX8-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v6, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4030,15 +4031,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4068,15 +4069,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -4096,15 +4097,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX10-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v7, v5
; GFX10-NEXT: v_mov_b32_e32 v6, v4
-; GFX10-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX10-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX10-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX10-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX10-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -4123,41 +4124,41 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dwordx2 v[6:7], v[0:1], off
+; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f64 v[6:7], v[2:3], v[2:3]
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX90A-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
-; GFX90A-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
+; GFX90A-NEXT: v_max_f64 v[2:3], v[4:5], v[4:5]
+; GFX90A-NEXT: v_min_f64 v[2:3], v[2:3], v[6:7]
+; GFX90A-NEXT: global_atomic_cmpswap_x2 v[2:3], v[0:1], v[2:5], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[6:7]
+; GFX90A-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[4:5]
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], v[4:5], v[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[4:5], v[2:3], v[2:3] op_sel:[0,1]
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v4
-; GFX90A-NEXT: v_mov_b32_e32 v1, v5
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, v3
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -4175,15 +4176,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4201,32 +4202,30 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: v_mov_b32_e32 v7, v1
-; GFX7-NEXT: v_mov_b32_e32 v6, v0
+; GFX7-NEXT: v_mov_b32_e32 v5, v1
+; GFX7-NEXT: v_mov_b32_e32 v4, v0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s4, s6
; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
-; GFX7-NEXT: v_mov_b32_e32 v5, v3
-; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
+; GFX7-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_max_f64 v[2:3], v[10:11], v[10:11]
-; GFX7-NEXT: v_min_f64 v[8:9], v[2:3], v[0:1]
-; GFX7-NEXT: v_mov_b32_e32 v2, v10
-; GFX7-NEXT: v_mov_b32_e32 v0, v8
-; GFX7-NEXT: v_mov_b32_e32 v1, v9
-; GFX7-NEXT: v_mov_b32_e32 v3, v11
-; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
+; GFX7-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX7-NEXT: v_min_f64 v[6:7], v[0:1], v[10:11]
+; GFX7-NEXT: v_mov_b32_e32 v2, v8
+; GFX7-NEXT: v_mov_b32_e32 v0, v6
+; GFX7-NEXT: v_mov_b32_e32 v1, v7
+; GFX7-NEXT: v_mov_b32_e32 v3, v9
+; GFX7-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX7-NEXT: s_waitcnt vmcnt(0)
; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; GFX7-NEXT: v_mov_b32_e32 v11, v1
+; GFX7-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX7-NEXT: v_mov_b32_e32 v9, v1
; GFX7-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v10, v0
+; GFX7-NEXT: v_mov_b32_e32 v8, v0
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB24_1
; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4237,33 +4236,30 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s6, 0
-; GFX6-NEXT: v_mov_b32_e32 v7, v1
-; GFX6-NEXT: v_mov_b32_e32 v6, v0
+; GFX6-NEXT: v_mov_b32_e32 v5, v1
+; GFX6-NEXT: v_mov_b32_e32 v4, v0
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s4, s6
; GFX6-NEXT: s_mov_b32 s5, s6
-; GFX6-NEXT: buffer_load_dwordx2 v[10:11], v[6:7], s[4:7], 0 addr64
-; GFX6-NEXT: v_mov_b32_e32 v5, v3
-; GFX6-NEXT: v_mov_b32_e32 v4, v2
+; GFX6-NEXT: buffer_load_dwordx2 v[8:9], v[4:5], s[4:7], 0 addr64
+; GFX6-NEXT: v_max_f64 v[10:11], v[2:3], v[2:3]
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_max_f64 v[2:3], v[10:11], v[10:11]
-; GFX6-NEXT: v_min_f64 v[8:9], v[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, v10
-; GFX6-NEXT: v_mov_b32_e32 v0, v8
-; GFX6-NEXT: v_mov_b32_e32 v1, v9
-; GFX6-NEXT: v_mov_b32_e32 v3, v11
-; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_max_f64 v[0:1], v[8:9], v[8:9]
+; GFX6-NEXT: v_min_f64 v[6:7], v[0:1], v[10:11]
+; GFX6-NEXT: v_mov_b32_e32 v2, v8
+; GFX6-NEXT: v_mov_b32_e32 v0, v6
+; GFX6-NEXT: v_mov_b32_e32 v1, v7
+; GFX6-NEXT: v_mov_b32_e32 v3, v9
+; GFX6-NEXT: buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
-; GFX6-NEXT: v_mov_b32_e32 v11, v1
+; GFX6-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v9, v1
; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v10, v0
+; GFX6-NEXT: v_mov_b32_e32 v8, v0
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB24_1
; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4283,15 +4279,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[8:9], v[6:7], v[6:7]
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[8:9], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4321,15 +4317,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX11-NEXT: v_max_f64 v[4:5], v[2:3], v[2:3]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f64 v[8:9], v[6:7], v[6:7]
-; GFX11-NEXT: v_min_f64 v[4:5], v[8:9], v[4:5]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
+; GFX11-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -4367,15 +4363,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dwordx2 v[4:5], v[0:1], off
+; GFX908-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX908-NEXT: s_mov_b64 s[4:5], 0
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v7, v5
; GFX908-NEXT: v_mov_b32_e32 v6, v4
-; GFX908-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX908-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX908-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX908-NEXT: global_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -4393,15 +4389,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx2 v[4:5], v[0:1]
+; GFX8-NEXT: v_max_f64 v[2:3], v[2:3], v[2:3]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v7, v5
; GFX8-NEXT: v_mov_b32_e32 v6, v4
-; GFX8-NEXT: v_max_f64 v[8:9], v[2:3], v[2:3]
; GFX8-NEXT: v_max_f64 v[4:5], v[6:7], v[6:7]
-; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[8:9]
+; GFX8-NEXT: v_min_f64 v[4:5], v[4:5], v[2:3]
; GFX8-NEXT: flat_atomic_cmpswap_x2 v[4:5], v[0:1], v[4:7] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4460,8 +4456,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4473,12 +4470,11 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4506,8 +4502,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4519,12 +4516,11 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4556,13 +4552,13 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_min_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -4584,8 +4580,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4597,12 +4594,11 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4626,8 +4622,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4639,12 +4636,11 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4668,6 +4664,7 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -4679,10 +4676,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4711,13 +4707,13 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_min_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
@@ -4745,14 +4741,14 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -4779,17 +4775,17 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -4896,33 +4892,35 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -4933,7 +4931,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4946,24 +4944,25 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -4990,67 +4989,69 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -5061,18 +5062,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5082,12 +5084,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5112,9 +5113,10 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -5123,10 +5125,9 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5146,36 +5147,36 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5191,14 +5192,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -5226,17 +5227,17 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -5347,33 +5348,35 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -5384,7 +5387,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5397,24 +5400,25 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5442,67 +5446,69 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB28_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -5513,18 +5519,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -5534,12 +5541,11 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -5564,9 +5570,10 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -5575,10 +5582,9 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5598,36 +5604,36 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB28_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5643,14 +5649,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -5678,17 +5684,17 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -5800,8 +5806,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5813,10 +5820,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5845,8 +5851,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5858,10 +5865,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5894,13 +5900,13 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
; GFX942-NEXT: v_not_b32_e32 v6, v4
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
; GFX942-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX942-NEXT: v_min_f16_e32 v4, v4, v2
; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
@@ -5921,8 +5927,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5934,10 +5941,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -5962,8 +5968,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5975,10 +5982,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
@@ -6003,6 +6009,7 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
@@ -6014,9 +6021,8 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX10-NEXT: v_min_f16_e32 v3, v3, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6045,13 +6051,13 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX90A-NEXT: v_not_b32_e32 v6, v4
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
; GFX90A-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
+; GFX90A-NEXT: v_min_f16_e32 v4, v4, v2
; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
@@ -6078,13 +6084,13 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX908-NEXT: v_not_b32_e32 v6, v3
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX908-NEXT: v_min_f16_e32 v3, v3, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
@@ -6111,16 +6117,16 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
; GFX8-NEXT: v_not_b32_e32 v6, v3
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB29_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
+; GFX8-NEXT: v_min_f16_e32 v3, v3, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v4, v6
; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v7, v3
; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -6226,36 +6232,38 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6272,36 +6280,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6318,30 +6327,30 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB30_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6351,37 +6360,39 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6393,37 +6404,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6435,31 +6446,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB30_1
@@ -6470,31 +6481,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB30_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6504,31 +6515,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB30_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6538,32 +6549,32 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB30_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6666,36 +6677,38 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6712,36 +6725,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6759,30 +6773,30 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB31_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6792,37 +6806,39 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6834,37 +6850,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6876,31 +6892,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, -1, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB31_1
@@ -6911,31 +6927,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB31_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6945,31 +6961,31 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0xfffff800, v0
; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, -1, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB31_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6979,32 +6995,32 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX8-LABEL: global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0xfffff800, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB31_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7108,15 +7124,15 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7143,15 +7159,15 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7173,44 +7189,44 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB32_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7233,15 +7249,15 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7264,14 +7280,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_min_f16_e32 v3, v5, v3
+; GFX10-NEXT: v_max_f16_e32 v3, v4, v4
+; GFX10-NEXT: v_min_f16_e32 v3, v3, v2
; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -7291,27 +7307,27 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB32_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -7319,14 +7335,14 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
; GFX908-NEXT: v_max_f16_e32 v3, v4, v4
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v5
+; GFX908-NEXT: v_min_f16_e32 v3, v3, v2
; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
@@ -7347,19 +7363,19 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v1, v2, v2
; GFX8-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v0, v1, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_e32 v0, v6, v6
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v6
+; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v5, v2, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB32_1
@@ -7448,14 +7464,14 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7481,24 +7497,24 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v2, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7511,24 +7527,24 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX942-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v4, v2, v2
; GFX942-NEXT: s_mov_b32 s2, 0xffff0000
; GFX942-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_max_f16_e32 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX942-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX942-NEXT: v_and_or_b32 v4, v5, s2, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX942-NEXT: v_and_or_b32 v2, v3, s2, v2
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB33_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7539,14 +7555,14 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v4.l, v4.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v4.l, v4.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
@@ -7568,25 +7584,25 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v4, v2, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v2, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7598,23 +7614,23 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX10-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2046
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2046
+; GFX10-NEXT: v_max_f16_e32 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_max_f16_e32 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX10-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX10-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB33_1
@@ -7625,22 +7641,22 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX90A-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2046
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v4, v2, v2
; GFX90A-NEXT: s_mov_b32 s6, 0xffff0000
; GFX90A-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_max_f16_e32 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_max_f16_e32 v4, v5, v5
-; GFX90A-NEXT: v_min_f16_e32 v3, v4, v3
-; GFX90A-NEXT: v_and_or_b32 v4, v5, s6, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2046 glc
+; GFX90A-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX90A-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB33_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7650,22 +7666,22 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX908-LABEL: global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2046
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2046
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v4, v2, v2
; GFX908-NEXT: s_mov_b32 s6, 0xffff0000
; GFX908-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_max_f16_e32 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX908-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, s6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX908-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX908-NEXT: v_and_or_b32 v2, v3, s6, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2046 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB33_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7677,22 +7693,22 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v4, v2, v2
; GFX8-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_min_f16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_e32 v2, v3, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB33_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7777,34 +7793,36 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v5.l, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
@@ -7815,7 +7833,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7828,24 +7846,25 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v7
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7873,67 +7892,69 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB34_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v5.l, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v5.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
@@ -7944,18 +7965,19 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -7965,12 +7987,11 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX11-FAKE16-NEXT: v_max_f16_e32 v5, v5, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v5, v5, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
@@ -7995,9 +8016,10 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: global_load_dword v5, v[0:1], off
; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
@@ -8006,10 +8028,9 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v6, v5
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
; GFX10-NEXT: v_lshrrev_b32_e32 v5, v3, v6
; GFX10-NEXT: v_max_f16_e32 v5, v5, v5
-; GFX10-NEXT: v_min_f16_e32 v5, v5, v7
+; GFX10-NEXT: v_min_f16_e32 v5, v5, v2
; GFX10-NEXT: v_lshlrev_b32_sdwa v5, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -8029,39 +8050,39 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX90A-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB34_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v3, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8077,14 +8098,14 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX908-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX908-NEXT: v_not_b32_e32 v4, v4
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
; GFX908-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v6, v5
-; GFX908-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX908-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX908-NEXT: v_min_f16_e32 v5, v7, v5
+; GFX908-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX908-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX908-NEXT: v_min_f16_e32 v5, v5, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v5, v3, v5
; GFX908-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX908-NEXT: global_atomic_cmpswap v5, v[0:1], v[5:6], off glc
@@ -8112,17 +8133,17 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX8-NEXT: v_lshlrev_b32_e64 v4, v3, s4
; GFX8-NEXT: v_not_b32_e32 v4, v4
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v6, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, v3, v6
-; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v7, v7
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v5
-; GFX8-NEXT: v_and_b32_e32 v8, v6, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v5
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_and_b32_e32 v7, v6, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v5, v8, v5
+; GFX8-NEXT: v_or_b32_e32 v5, v7, v5
; GFX8-NEXT: flat_atomic_cmpswap v5, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
@@ -8233,37 +8254,39 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.l, v2.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v3.l, v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v5.l, v5.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8280,37 +8303,38 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v7, v2, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v7
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8327,30 +8351,30 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v4, v3, s0
-; GFX942-NEXT: v_not_b32_e32 v6, v4
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_max_f16_e32 v6, v2, v2
; GFX942-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX942-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX942-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX942-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX942-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX942-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX942-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX942-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB35_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8360,37 +8384,39 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v2.h, v2.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v3.l, v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_min_f16_e32 v2.h, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8402,37 +8428,37 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-FAKE16-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX11-FAKE16-NEXT: v_min_f16_e32 v3, v3, v7
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX11-FAKE16-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8444,31 +8470,31 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX10-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX10-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT: v_max_f16_e32 v6, v2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v4
; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v5, v5
; GFX10-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX10-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX10-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX10-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX10-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX10-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB35_1
@@ -8479,34 +8505,34 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX90A-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
+; GFX90A-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
; GFX90A-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX90A-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
-; GFX90A-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX90A-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
+; GFX90A-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX90A-NEXT: s_mov_b32 s4, 0xffff
-; GFX90A-NEXT: v_lshlrev_b32_e64 v4, v3, s4
-; GFX90A-NEXT: v_not_b32_e32 v6, v4
+; GFX90A-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX90A-NEXT: v_not_b32_e32 v5, v5
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_max_f16_e32 v6, v2, v2
; GFX90A-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX90A-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX90A-NEXT: v_max_f16_e32 v4, v4, v4
-; GFX90A-NEXT: v_min_f16_e32 v4, v4, v7
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX90A-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX90A-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX90A-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX90A-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX90A-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v4
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB35_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8515,32 +8541,32 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
;
; GFX908-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: v_add_co_u32_e32 v3, vcc, 0x7fe, v0
-; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX908-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
-; GFX908-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX908-NEXT: v_add_co_u32_e32 v4, vcc, 0x7fe, v0
+; GFX908-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX908-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
+; GFX908-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX908-NEXT: s_mov_b32 s4, 0xffff
-; GFX908-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX908-NEXT: v_not_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX908-NEXT: v_not_b32_e32 v5, v5
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_max_f16_e32 v6, v2, v2
; GFX908-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX908-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX908-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX908-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX908-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX908-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX908-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX908-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB35_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8550,32 +8576,32 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX8-LABEL: global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 0x7fe, v0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 0x7fe, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX8-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 3, v4
; GFX8-NEXT: s_mov_b32 s4, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v5, s4
-; GFX8-NEXT: v_not_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v4, s4
+; GFX8-NEXT: v_not_b32_e32 v5, v5
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
; GFX8-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX8-NEXT: v_max_f16_e32 v7, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v3, v3, v3
-; GFX8-NEXT: v_min_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_and_b32_e32 v8, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
+; GFX8-NEXT: v_min_f16_e32 v2, v2, v6
+; GFX8-NEXT: v_and_b32_e32 v7, v3, v5
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB35_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -12634,15 +12660,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12661,43 +12687,43 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB46_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -12717,14 +12743,14 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -12742,25 +12768,25 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB46_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memory:
@@ -12768,13 +12794,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -12792,21 +12818,21 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v3, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_max_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB46_1
@@ -12916,15 +12942,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12943,43 +12969,43 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB47_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -12999,14 +13025,14 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -13024,25 +13050,25 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB47_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -13050,13 +13076,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13076,21 +13102,21 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB47_1
@@ -13200,15 +13226,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13227,43 +13253,43 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB48_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -13283,14 +13309,14 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -13308,25 +13334,25 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB48_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -13334,13 +13360,13 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -13360,21 +13386,21 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB48_1
@@ -13491,21 +13517,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13518,23 +13544,23 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB49_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13544,22 +13570,22 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13571,21 +13597,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB49_1
@@ -13596,20 +13622,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB49_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13619,20 +13645,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB49_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13642,24 +13668,24 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX8-LABEL: global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB49_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13760,21 +13786,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13787,23 +13813,23 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13813,22 +13839,22 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -13840,21 +13866,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -13865,20 +13891,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13888,20 +13914,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -13913,24 +13939,24 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14032,21 +14058,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14059,23 +14085,23 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14085,22 +14111,22 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX11-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14112,21 +14138,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -14137,20 +14163,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14160,20 +14186,20 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB51_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14185,24 +14211,24 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14313,15 +14339,15 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14341,43 +14367,43 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX942-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v2
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -14397,14 +14423,14 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v4, v4
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v2
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -14422,28 +14448,28 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX90A-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -14451,13 +14477,13 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v2
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_pk_max_f16 v5, v2, v2
; GFX908-NEXT: v_pk_max_f16 v3, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v3, v5
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v2
; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
@@ -14477,21 +14503,21 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v2, v2, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_max_f16_sdwa v5, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v6, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v7, v1, v1
-; GFX8-NEXT: v_min_f16_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v0, v7, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v6, v6
+; GFX8-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v5, v5, v0
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -14600,22 +14626,22 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v5, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v5, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14628,23 +14654,23 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX942-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v5, v5
+; GFX942-NEXT: v_pk_max_f16 v2, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v2, v2, v4
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14654,22 +14680,22 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX11-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v5, v4, v4
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v4
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-NEXT: v_mov_b32_e32 v3, v2
; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -14681,21 +14707,21 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX10-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX10-NEXT: v_pk_min_f16 v3, v5, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s4
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -14706,23 +14732,23 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX90A-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v5, v5
-; GFX90A-NEXT: v_pk_min_f16 v4, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v2, v2, v4
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14732,20 +14758,20 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX908-LABEL: global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v5, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v3, v5, v3
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v4
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14757,24 +14783,24 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v2, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_e32 v7, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v5, v7, v6
-; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_max_f16_sdwa v2, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v6, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v6, v6, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -14881,41 +14907,43 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14934,38 +14962,39 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14979,87 +15008,88 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15074,39 +15104,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15121,34 +15153,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -15160,41 +15192,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_memory:
@@ -15202,33 +15234,33 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
@@ -15242,34 +15274,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
@@ -15390,41 +15422,43 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15443,38 +15477,39 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15488,87 +15523,88 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15583,39 +15619,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15630,34 +15668,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -15669,41 +15707,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -15711,33 +15749,33 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB55_1
@@ -15753,34 +15791,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
@@ -15901,41 +15939,43 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15954,38 +15994,39 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15999,87 +16040,88 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX942-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16094,39 +16136,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16141,34 +16185,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -16180,41 +16224,41 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX90A-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -16222,33 +16266,33 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -16264,34 +16308,34 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -16419,43 +16463,42 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16472,40 +16515,40 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16518,39 +16561,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16560,44 +16603,44 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16610,41 +16653,41 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16657,35 +16700,35 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -16696,36 +16739,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16735,36 +16778,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16774,37 +16817,37 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX8-LABEL: global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16914,43 +16957,42 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16967,40 +17009,40 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17013,39 +17055,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB58_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17055,44 +17097,44 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17105,41 +17147,41 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17152,35 +17194,35 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB58_1
@@ -17191,36 +17233,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB58_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17230,36 +17272,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB58_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17271,37 +17313,37 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB58_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17412,43 +17454,42 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17465,40 +17506,40 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17511,39 +17552,39 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX942-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB59_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17553,44 +17594,44 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17603,41 +17644,41 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17650,35 +17691,35 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX10-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB59_1
@@ -17689,36 +17730,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX90A-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB59_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17728,36 +17769,36 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX908-LABEL: global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB59_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17769,37 +17810,37 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB59_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17919,42 +17960,44 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17973,39 +18016,40 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18019,87 +18063,88 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX942-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB60_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18114,39 +18159,41 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18161,34 +18208,34 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB60_1
@@ -18200,44 +18247,44 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX90A-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB60_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -18245,33 +18292,33 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_min_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_min_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB60_1
@@ -18287,34 +18334,34 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_min_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_min_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB60_1
@@ -18434,44 +18481,43 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18488,41 +18534,41 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18535,39 +18581,39 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX942-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB61_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18577,44 +18623,44 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18627,41 +18673,41 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-FAKE16-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18674,35 +18720,35 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX10-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB61_1
@@ -18713,39 +18759,39 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX90A-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB61_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18755,36 +18801,36 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX908-LABEL: global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB61_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18796,37 +18842,37 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_min_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB61_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index db4f52a1963ed..ef314c8f68b1f 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -15265,41 +15265,43 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15318,38 +15320,39 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15363,87 +15366,88 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX942-LABEL: global_agent_atomic_fsub_ret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB50_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15458,39 +15462,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -15505,34 +15511,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB50_1
@@ -15544,41 +15550,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB50_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16:
@@ -15586,33 +15592,33 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB50_1
@@ -15626,34 +15632,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16(ptr addrspace(1) %ptr,
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v3
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v3
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v3, 16
+; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB50_1
@@ -15774,41 +15780,43 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15827,38 +15835,39 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15872,87 +15881,88 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX942-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB51_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -15967,39 +15977,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16014,34 +16026,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB51_1
@@ -16053,41 +16065,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB51_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -16095,33 +16107,33 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB51_1
@@ -16137,34 +16149,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_pos(ptr addr
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB51_1
@@ -16285,41 +16297,43 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16338,38 +16352,39 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16383,87 +16398,88 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX942-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB52_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16478,39 +16494,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -16525,34 +16543,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB52_1
@@ -16564,41 +16582,41 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX90A-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB52_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_agent_atomic_fsub_ret_v2bf16__offset12b_neg:
@@ -16606,33 +16624,33 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB52_1
@@ -16648,34 +16666,34 @@ define <2 x bfloat> @global_agent_atomic_fsub_ret_v2bf16__offset12b_neg(ptr addr
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, -1, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB52_1
@@ -16803,43 +16821,42 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16856,40 +16873,40 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16902,39 +16919,39 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX942-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB53_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -16944,44 +16961,44 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -16994,41 +17011,41 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17041,35 +17058,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: global_load_dword v3, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB53_1
@@ -17080,36 +17097,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB53_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17119,36 +17136,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off
+; GFX908-NEXT: global_load_dword v3, v[0:1], off
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB53_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17158,37 +17175,37 @@ define void @global_agent_atomic_fsub_noret_v2bf16(ptr addrspace(1) %ptr, <2 x b
; GFX8-LABEL: global_agent_atomic_fsub_noret_v2bf16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB53_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17298,43 +17315,42 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17351,40 +17367,40 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17397,39 +17413,39 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX942-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB54_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17439,44 +17455,44 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17489,41 +17505,41 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -17536,35 +17552,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB54_1
@@ -17575,36 +17591,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB54_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17614,36 +17630,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB54_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17655,37 +17671,37 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace(
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB54_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17796,43 +17812,42 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17849,40 +17864,40 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -17895,39 +17910,39 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX942-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 sc0
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB55_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -17937,44 +17952,44 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -17987,41 +18002,41 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18034,35 +18049,35 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX10-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:-2048
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB55_1
@@ -18073,36 +18088,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX90A-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:-2048
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:-2048 glc
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB55_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18112,36 +18127,36 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX908-LABEL: global_agent_atomic_fsub_noret_v2bf16__offset12b_neg:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:-2048
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:-2048
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:-2048 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB55_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18153,37 +18168,37 @@ define void @global_agent_atomic_fsub_noret_v2bf16__offset12b_neg(ptr addrspace(
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0xfffff800, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, -1, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB55_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18303,42 +18318,44 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18357,39 +18374,40 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18403,87 +18421,88 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX942-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB56_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v3
+; GFX942-NEXT: v_mov_b32_e32 v0, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_and_b32 v3, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -18498,39 +18517,41 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_lshlrev_b32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v7, v5 :: v_dual_lshlrev_b32 v6, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v7, v9, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -18545,34 +18566,34 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v6, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v5
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_mov_b32_e32 v6, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX10-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v5, v5, v3, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v6
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB56_1
@@ -18584,44 +18605,44 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX90A-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB56_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v3
+; GFX90A-NEXT: v_mov_b32_e32 v0, v2
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: global_system_atomic_fsub_ret_v2bf16__offset12b_pos:
@@ -18629,33 +18650,33 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX908-NEXT: v_sub_f32_e32 v3, v7, v3
-; GFX908-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX908-NEXT: v_add3_u32 v6, v6, v5, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v3, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v3, v3
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_mov_b32_e32 v6, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX908-NEXT: v_sub_f32_e32 v3, v3, v4
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX908-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v7, v7, v3, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
+; GFX908-NEXT: v_cndmask_b32_e64 v3, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v5, v5, v3, s9
+; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[5:6], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v6
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB56_1
@@ -18671,34 +18692,34 @@ define <2 x bfloat> @global_system_atomic_fsub_ret_v2bf16__offset12b_pos(ptr add
; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v0, v[3:4]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
; GFX8-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v0, 0xffff0000, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
-; GFX8-NEXT: v_sub_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_sub_f32_e32 v0, v7, v0
-; GFX8-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v5
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v0
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v5, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v6, v7, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v6, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v6
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX8-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX8-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v0
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v5
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v8, s[4:5]
; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v0, v5, v0, 16
-; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[0:1] glc
+; GFX8-NEXT: v_alignbit_b32 v5, v5, v0, 16
+; GFX8-NEXT: flat_atomic_cmpswap v0, v[3:4], v[5:6] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v0, v6
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB56_1
@@ -18818,44 +18839,43 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18872,41 +18892,41 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -18919,39 +18939,39 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX942-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX942-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX942-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v6, v6, v3, s4
-; GFX942-NEXT: v_add3_u32 v8, v8, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX942-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX942-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX942-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX942-NEXT: v_add3_u32 v7, v7, v2, s4
+; GFX942-NEXT: v_add3_u32 v9, v9, v6, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[0:1]
-; GFX942-NEXT: v_perm_b32 v4, v4, v3, s5
+; GFX942-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[0:1]
+; GFX942-NEXT: v_perm_b32 v2, v6, v2, s5
; GFX942-NEXT: buffer_wbl2 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 sc0 sc1
+; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: buffer_inv sc0 sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB57_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -18961,44 +18981,44 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-TRUE16-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -19011,41 +19031,41 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX11-FAKE16-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v6, v8, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v7, v9, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl1_inv
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -19058,35 +19078,35 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX10-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX10-NEXT: global_load_dword v3, v[0:1], off offset:2044
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX10-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX10-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX10-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX10-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
-; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v8, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v5, v3, 0x7060302
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX10-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX10-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX10-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v7, v9, s4
+; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX10-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: buffer_gl1_inv
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v2
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB57_1
@@ -19097,39 +19117,39 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX90A-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: global_load_dword v5, v[0:1], off offset:2044
+; GFX90A-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v5
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v7, v6
-; GFX90A-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX90A-NEXT: v_add3_u32 v8, v8, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc
-; GFX90A-NEXT: v_perm_b32 v4, v4, v3, s9
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX90A-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX90A-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX90A-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX90A-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX90A-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX90A-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX90A-NEXT: v_perm_b32 v2, v6, v2, s9
; GFX90A-NEXT: buffer_wbl2
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_atomic_cmpswap v3, v[0:1], v[4:5], off offset:2044 glc
+; GFX90A-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: buffer_invl2
; GFX90A-NEXT: buffer_wbinvl1
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v5
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v5, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB57_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19139,36 +19159,36 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX908-LABEL: global_system_atomic_fsub_noret_v2bf16__offset12b_pos:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: global_load_dword v4, v[0:1], off offset:2044
+; GFX908-NEXT: global_load_dword v3, v[0:1], off offset:2044
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX908-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX908-NEXT: v_add3_u32 v6, v6, v3, s8
-; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX908-NEXT: v_perm_b32 v3, v5, v3, s9
-; GFX908-NEXT: global_atomic_cmpswap v3, v[0:1], v[3:4], off offset:2044 glc
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX908-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
+; GFX908-NEXT: v_add3_u32 v9, v9, v6, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX908-NEXT: v_perm_b32 v2, v6, v2, s9
+; GFX908-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off offset:2044 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: buffer_wbinvl1
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v4, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB57_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -19180,37 +19200,37 @@ define void @global_system_atomic_fsub_noret_v2bf16__offset12b_pos(ptr addrspace
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 0x7fc, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
+; GFX8-NEXT: flat_load_dword v3, v[0:1]
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX8-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_and_b32_e32 v7, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v5, v7, v6
-; GFX8-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v3
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
-; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
-; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_alignbit_b32 v3, v5, v3, 16
-; GFX8-NEXT: flat_atomic_cmpswap v3, v[0:1], v[3:4] glc
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v4
+; GFX8-NEXT: v_sub_f32_e32 v6, v6, v5
+; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, v9, v6
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
+; GFX8-NEXT: v_add_u32_e32 v9, vcc, 0x7fff, v9
+; GFX8-NEXT: v_or_b32_e32 v10, 0x400000, v6
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v6, v6
+; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v7, v8, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX8-NEXT: v_alignbit_b32 v2, v6, v2, 16
+; GFX8-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: buffer_wbinvl1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v2
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB57_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
index 177e75c865e92..3fa6e00380132 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-saddr-to-vaddr.ll
@@ -56,19 +56,19 @@ define amdgpu_kernel void @test_move_load_address_to_vgpr_d16_hi(ptr addrspace(1
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_movk_i32 s2, 0x100
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_load_ushort v0, v1, s[0:1] glc
; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: s_movk_i32 s1, 0x100
; GCN-NEXT: .LBB1_1: ; %bb3
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: v_lshlrev_b64 v[2:3], 1, v[0:1]
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, s0, v2
-; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v0, v3, vcc
-; GCN-NEXT: global_load_short_d16_hi v0, v[2:3], off glc
+; GCN-NEXT: v_lshlrev_b64 v[3:4], 1, v[0:1]
+; GCN-NEXT: v_add_co_u32_e32 v3, vcc, s0, v3
+; GCN-NEXT: v_addc_co_u32_e32 v4, vcc, v2, v4, vcc
+; GCN-NEXT: global_load_short_d16_hi v0, v[3:4], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
+; GCN-NEXT: v_cmp_eq_u32_e32 vcc, s1, v0
; GCN-NEXT: s_cbranch_vccz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %bb2
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index 8ea260ca64ff4..94be01dfd0976 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -1632,20 +1632,18 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB34_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_subrev_u32_e32 v2, vcc, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB34_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -1724,24 +1722,22 @@ define amdgpu_gfx void @global_atomic_sub_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB35_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_subrev_u32_e32 v2, vcc, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB35_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_sub_i32_noret_offset_scalar:
@@ -1817,17 +1813,17 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB36_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_subrev_u32_e32 v3, vcc, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB36_1
@@ -1905,26 +1901,24 @@ define amdgpu_gfx i32 @global_atomic_sub_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB37_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_subrev_u32_e32 v3, vcc, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB37_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_sub_i32_ret_offset_scalar:
@@ -2423,20 +2417,18 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB45_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB45_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -2515,24 +2507,22 @@ define amdgpu_gfx void @global_atomic_and_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB46_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB46_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_and_i32_noret_offset_scalar:
@@ -2608,17 +2598,17 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB47_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB47_1
@@ -2696,26 +2686,24 @@ define amdgpu_gfx i32 @global_atomic_and_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB48_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB48_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_and_i32_ret_offset_scalar:
@@ -3190,21 +3178,19 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB55_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_not_b32_e32 v0, v0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: v_not_b32_e32 v2, v2
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB55_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -3285,25 +3271,23 @@ define amdgpu_gfx void @global_atomic_nand_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB56_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_not_b32_e32 v0, v0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_and_b32_e32 v2, s6, v3
+; VI-NEXT: v_not_b32_e32 v2, v2
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB56_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i32_noret_offset_scalar:
@@ -3381,18 +3365,18 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB57_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_not_b32_e32 v0, v0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v0, s6, v4
+; VI-NEXT: v_not_b32_e32 v3, v0
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB57_1
@@ -3472,27 +3456,25 @@ define amdgpu_gfx i32 @global_atomic_nand_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB58_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_and_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_not_b32_e32 v0, v0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_and_b32_e32 v0, s6, v4
+; VI-NEXT: v_not_b32_e32 v3, v0
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB58_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i32_ret_offset_scalar:
@@ -4046,20 +4028,18 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_scalar(ptr addrspace(1) inreg
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB65_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_or_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB65_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4138,24 +4118,22 @@ define amdgpu_gfx void @global_atomic_or_i32_noret_offset_scalar(ptr addrspace(1
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB66_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_or_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_or_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB66_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_or_i32_noret_offset_scalar:
@@ -4231,17 +4209,17 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_scalar(ptr addrspace(1) inreg %p
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB67_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_or_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_or_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB67_1
@@ -4319,26 +4297,24 @@ define amdgpu_gfx i32 @global_atomic_or_i32_ret_offset_scalar(ptr addrspace(1) i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB68_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_or_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_or_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB68_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_or_i32_ret_offset_scalar:
@@ -4837,20 +4813,18 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB76_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_xor_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_xor_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB76_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -4929,24 +4903,22 @@ define amdgpu_gfx void @global_atomic_xor_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB77_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_xor_b32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_xor_b32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB77_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_xor_i32_noret_offset_scalar:
@@ -5022,17 +4994,17 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB78_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_xor_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_xor_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB78_1
@@ -5110,26 +5082,24 @@ define amdgpu_gfx i32 @global_atomic_xor_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB79_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_xor_b32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_xor_b32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB79_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_xor_i32_ret_offset_scalar:
@@ -5628,20 +5598,18 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB87_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB87_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5720,24 +5688,22 @@ define amdgpu_gfx void @global_atomic_max_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB88_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB88_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i32_noret_offset_scalar:
@@ -5813,17 +5779,17 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB89_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_i32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB89_1
@@ -5901,26 +5867,24 @@ define amdgpu_gfx i32 @global_atomic_max_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB90_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_i32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB90_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i32_ret_offset_scalar:
@@ -5989,26 +5953,26 @@ define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB91_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_max_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_i32_e32 v2, s2, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB91_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -6095,32 +6059,32 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB92_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB92_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_ret_addr64_offset:
@@ -6202,24 +6166,24 @@ define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in,
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: .LBB93_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_max_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_i32_e32 v2, s2, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB93_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -6305,30 +6269,30 @@ define amdgpu_kernel void @atomic_max_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: .LBB94_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB94_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_ret_addr64:
@@ -6802,20 +6766,18 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB101_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB101_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6894,24 +6856,22 @@ define amdgpu_gfx void @global_atomic_umax_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB102_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB102_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i32_noret_offset_scalar:
@@ -6987,17 +6947,17 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB103_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_u32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB103_1
@@ -7075,26 +7035,24 @@ define amdgpu_gfx i32 @global_atomic_umax_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB104_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_max_u32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB104_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i32_ret_offset_scalar:
@@ -7163,26 +7121,26 @@ define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out,
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB105_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_max_u32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_max_u32_e32 v2, s2, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB105_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -7269,32 +7227,32 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64_offset(ptr addrspace(1) %o
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB106_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB106_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_ret_addr64_offset:
@@ -7384,30 +7342,30 @@ define amdgpu_kernel void @atomic_umax_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: .LBB107_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_max_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_max_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB107_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_ret_addr64:
@@ -7881,20 +7839,18 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB114_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB114_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7973,24 +7929,22 @@ define amdgpu_gfx void @global_atomic_umin_i32_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB115_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_u32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB115_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i32_noret_offset_scalar:
@@ -8066,17 +8020,17 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB116_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_u32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB116_1
@@ -8154,26 +8108,24 @@ define amdgpu_gfx i32 @global_atomic_umin_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB117_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_min_u32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_u32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB117_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i32_ret_offset_scalar:
@@ -8635,20 +8587,18 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB124_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB124_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8727,24 +8677,22 @@ define amdgpu_gfx void @global_atomic_min_i32_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB125_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB125_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i32_noret_offset_scalar:
@@ -8820,17 +8768,17 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB126_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_i32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB126_1
@@ -8908,26 +8856,24 @@ define amdgpu_gfx i32 @global_atomic_min_i32_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB127_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_min_i32_e32 v3, s6, v4
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB127_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i32_ret_offset_scalar:
@@ -8996,26 +8942,26 @@ define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i
; VI-NEXT: s_ashr_i32 s5, s3, 31
; VI-NEXT: s_mov_b32 s4, s3
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s3, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB128_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_min_i32_e32 v0, s2, v1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_i32_e32 v2, s2, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB128_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -9102,32 +9048,32 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64_offset(ptr addrspace(1) %ou
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x10
-; VI-NEXT: s_add_u32 s0, s0, 16
-; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x10
+; VI-NEXT: s_add_u32 s4, s4, 16
+; VI-NEXT: s_addc_u32 s5, s5, 0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: .LBB129_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB129_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_ret_addr64_offset:
@@ -9200,25 +9146,25 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
;
; VI-LABEL: atomic_min_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
-; VI-NEXT: s_mov_b64 s[2:3], 0
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_mov_b64 s[0:1], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_load_dword s5, s[0:1], 0x0
+; VI-NEXT: s_load_dword s3, s[6:7], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: .LBB130_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_min_i32_e32 v0, s4, v1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_min_i32_e32 v2, s2, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB130_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -9299,30 +9245,30 @@ define amdgpu_kernel void @atomic_min_i32_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_ashr_i32 s5, s7, 31
; VI-NEXT: s_mov_b32 s4, s7
; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dword s7, s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dword s7, s[4:5], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: v_mov_b32_e32 v2, s7
; VI-NEXT: .LBB131_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_min_i32_e32 v0, s6, v1
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: v_min_i32_e32 v2, s6, v3
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB131_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[4:5]
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_ret_addr64:
@@ -9822,22 +9768,20 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_scalar(ptr addrspace(1
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: flat_load_dword v1, v[0:1]
+; VI-NEXT: flat_load_dword v3, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB138_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_add_u32_e32 v2, vcc, 1, v3
+; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; VI-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v1, v0
+; VI-NEXT: v_mov_b32_e32 v3, v2
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB138_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9920,26 +9864,24 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i32_noret_offset_scalar(ptr addr
; VI-NEXT: s_addc_u32 s35, s5, 0
; VI-NEXT: v_mov_b32_e32 v0, s34
; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB139_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_add_u32_e32 v2, vcc, 1, v3
+; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v3
+; VI-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB139_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_uinc_wrap_i32_noret_offset_scalar:
@@ -10019,19 +9961,19 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_scalar(ptr addrspace(1) i
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB140_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v4
+; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; VI-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB140_1
@@ -10113,28 +10055,26 @@ define amdgpu_gfx i32 @global_atomic_uinc_wrap_i32_ret_offset_scalar(ptr addrspa
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 16
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB141_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v1
-; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v1
-; VI-NEXT: v_mov_b32_e32 v2, s34
-; VI-NEXT: v_mov_b32_e32 v3, s35
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_mov_b32_e32 v4, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v4
+; VI-NEXT: v_cmp_gt_u32_e32 vcc, s6, v4
+; VI-NEXT: v_cndmask_b32_e32 v3, 0, v0, vcc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[3:4] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB141_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_uinc_wrap_i32_ret_offset_scalar:
@@ -10594,41 +10534,41 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v4, s6, 0
-; SI-NEXT: v_writelane_b32 v4, s7, 1
+; SI-NEXT: v_writelane_b32 v5, s6, 0
+; SI-NEXT: v_writelane_b32 v5, s7, 1
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: .LBB148_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v1
; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v3, v1
-; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; SI-NEXT: v_mov_b32_e32 v1, v2
+; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB148_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v4, 1
-; SI-NEXT: v_readlane_b32 s6, v4, 0
+; SI-NEXT: v_readlane_b32 s7, v5, 1
+; SI-NEXT: v_readlane_b32 s6, v5, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -10640,15 +10580,15 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v1, v[0:1]
; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB148_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v1
; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; VI-NEXT: v_mov_b32_e32 v4, s6
; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -10668,12 +10608,12 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_scalar(ptr addrspace(1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: .LBB148_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v1
; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[4:5] glc
@@ -10696,41 +10636,41 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v4, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v4, s6, 0
-; SI-NEXT: v_writelane_b32 v4, s7, 1
+; SI-NEXT: v_writelane_b32 v5, s6, 0
+; SI-NEXT: v_writelane_b32 v5, s7, 1
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v1, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: .LBB149_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v1
; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v3, v1
-; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 offset:16 glc
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v4, v1
+; SI-NEXT: v_mov_b32_e32 v3, v0
+; SI-NEXT: buffer_atomic_cmpswap v[3:4], off, s[4:7], 0 offset:16 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v3, v1
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; SI-NEXT: v_mov_b32_e32 v1, v2
+; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB149_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v4, 1
-; SI-NEXT: v_readlane_b32 s6, v4, 0
+; SI-NEXT: v_readlane_b32 s7, v5, 1
+; SI-NEXT: v_readlane_b32 s6, v5, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v4, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -10738,32 +10678,30 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addr
; VI-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: s_add_u32 s36, s4, 16
-; VI-NEXT: s_addc_u32 s37, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s36
-; VI-NEXT: v_mov_b32_e32 v1, s37
-; VI-NEXT: flat_load_dword v1, v[0:1]
-; VI-NEXT: s_mov_b64 s[38:39], 0
+; VI-NEXT: s_add_u32 s34, s4, 16
+; VI-NEXT: s_addc_u32 s35, s5, 0
+; VI-NEXT: v_mov_b32_e32 v0, s34
+; VI-NEXT: v_mov_b32_e32 v1, s35
+; VI-NEXT: flat_load_dword v3, v[0:1]
+; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s6
; VI-NEXT: .LBB149_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v1
-; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; VI-NEXT: v_mov_b32_e32 v4, s6
+; VI-NEXT: v_subrev_u32_e32 v2, vcc, 1, v3
+; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v3
; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v2, s36
-; VI-NEXT: v_mov_b32_e32 v3, s37
-; VI-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; VI-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: v_mov_b32_e32 v3, v2
+; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB149_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[38:39]
+; VI-NEXT: s_or_b64 exec, exec, s[36:37]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_udec_wrap_i32_noret_offset_scalar:
@@ -10772,12 +10710,12 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i32_noret_offset_scalar(ptr addr
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dword v1, v2, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: .LBB149_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v1
; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[4:5] offset:16 glc
@@ -10801,41 +10739,41 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v3, s6, 0
-; SI-NEXT: v_writelane_b32 v3, s7, 1
+; SI-NEXT: v_writelane_b32 v5, s6, 0
+; SI-NEXT: v_writelane_b32 v5, s7, 1
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: .LBB150_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: v_mov_b32_e32 v4, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v1, vcc, 1, v2
-; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v2
-; SI-NEXT: v_mov_b32_e32 v0, s34
+; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v4
+; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v4
; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; SI-NEXT: v_mov_b32_e32 v0, v1
-; SI-NEXT: v_mov_b32_e32 v1, v2
+; SI-NEXT: v_cndmask_b32_e32 v3, v0, v2, vcc
+; SI-NEXT: v_mov_b32_e32 v0, v3
+; SI-NEXT: v_mov_b32_e32 v1, v4
; SI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB150_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v3, 1
-; SI-NEXT: v_readlane_b32 s6, v3, 0
+; SI-NEXT: v_readlane_b32 s7, v5, 1
+; SI-NEXT: v_readlane_b32 s6, v5, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -10847,21 +10785,21 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) i
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dword v0, v[0:1]
; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v3, s6
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v2, s5
; VI-NEXT: .LBB150_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_subrev_u32_e32 v4, vcc, 1, v1
-; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: v_mov_b32_e32 v5, v0
+; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v5
+; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB150_1
@@ -10875,19 +10813,19 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_scalar(ptr addrspace(1) i
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: .LBB150_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, 1, v3
-; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v3
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v4
+; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v4
; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[4:5] glc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v0, v2, vcc
+; GFX9-NEXT: global_atomic_cmpswap v0, v1, v[3:4], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB150_1
@@ -10903,41 +10841,41 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspa
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v5, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v3, s6, 0
-; SI-NEXT: v_writelane_b32 v3, s7, 1
+; SI-NEXT: v_writelane_b32 v5, s6, 0
+; SI-NEXT: v_writelane_b32 v5, s7, 1
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v2, s34
; SI-NEXT: .LBB151_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, v0
+; SI-NEXT: v_mov_b32_e32 v4, v0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v1, vcc, 1, v2
-; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v2
-; SI-NEXT: v_mov_b32_e32 v0, s34
+; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v4
+; SI-NEXT: v_cmp_lt_u32_e64 s[36:37], s34, v4
; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
-; SI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; SI-NEXT: v_mov_b32_e32 v0, v1
-; SI-NEXT: v_mov_b32_e32 v1, v2
+; SI-NEXT: v_cndmask_b32_e32 v3, v0, v2, vcc
+; SI-NEXT: v_mov_b32_e32 v0, v3
+; SI-NEXT: v_mov_b32_e32 v1, v4
; SI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v2
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB151_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v3, 1
-; SI-NEXT: v_readlane_b32 s6, v3, 0
+; SI-NEXT: v_readlane_b32 s7, v5, 1
+; SI-NEXT: v_readlane_b32 s6, v5, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v3, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v5, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -10945,32 +10883,30 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspa
; VI-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: s_add_u32 s36, s4, 16
-; VI-NEXT: s_addc_u32 s37, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s36
-; VI-NEXT: v_mov_b32_e32 v1, s37
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_mov_b64 s[38:39], 0
+; VI-NEXT: s_add_u32 s34, s4, 16
+; VI-NEXT: s_addc_u32 s35, s5, 0
+; VI-NEXT: v_mov_b32_e32 v1, s34
+; VI-NEXT: v_mov_b32_e32 v2, s35
+; VI-NEXT: flat_load_dword v0, v[1:2]
+; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v3, s6
; VI-NEXT: .LBB151_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: v_subrev_u32_e32 v4, vcc, 1, v1
-; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v1
-; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: v_mov_b32_e32 v5, v0
+; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v5
+; VI-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v5
; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v2, s36
-; VI-NEXT: v_mov_b32_e32 v3, s37
-; VI-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
+; VI-NEXT: v_cndmask_b32_e32 v4, v0, v3, vcc
+; VI-NEXT: flat_atomic_cmpswap v0, v[1:2], v[4:5] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, v0, v5
+; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB151_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[38:39]
+; VI-NEXT: s_or_b64 exec, exec, s[36:37]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_udec_wrap_i32_ret_offset_scalar:
@@ -10979,19 +10915,19 @@ define amdgpu_gfx i32 @global_atomic_udec_wrap_i32_ret_offset_scalar(ptr addrspa
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: global_load_dword v0, v1, s[4:5] offset:16
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: .LBB151_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, 1, v3
-; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v3
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v4
+; GFX9-NEXT: v_cmp_lt_u32_e64 s[34:35], s6, v4
; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[4:5] offset:16 glc
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v0, v2, vcc
+; GFX9-NEXT: global_atomic_cmpswap v0, v1, v[3:4], s[4:5] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v4
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB151_1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
index b2883b43e2b49..d9d77e677c43e 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
@@ -1678,43 +1678,43 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
-; SI-NEXT: s_mov_b32 s34, s7
-; SI-NEXT: s_mov_b32 s35, s6
+; SI-NEXT: v_writelane_b32 v9, s6, 0
+; SI-NEXT: v_writelane_b32 v9, s7, 1
+; SI-NEXT: s_mov_b32 s35, s7
+; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
; SI-NEXT: .LBB34_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, s35, v2
-; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; SI-NEXT: v_subrev_i32_e32 v0, vcc, s34, v2
+; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v4, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v7, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v5, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v5
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v6
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB34_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v9, 1
+; SI-NEXT: v_readlane_b32 s6, v9, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -1726,14 +1726,14 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB34_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -1753,12 +1753,12 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1780,43 +1780,43 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v9, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
-; SI-NEXT: s_mov_b32 s34, s7
-; SI-NEXT: s_mov_b32 s35, s6
+; SI-NEXT: v_writelane_b32 v9, s6, 0
+; SI-NEXT: v_writelane_b32 v9, s7, 1
+; SI-NEXT: s_mov_b32 s35, s7
+; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
; SI-NEXT: .LBB35_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v1, s34
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, s35, v2
-; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; SI-NEXT: v_subrev_i32_e32 v0, vcc, s34, v2
+; SI-NEXT: v_subb_u32_e32 v1, vcc, v3, v4, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v8, v3
+; SI-NEXT: v_mov_b32_e32 v7, v2
+; SI-NEXT: v_mov_b32_e32 v6, v1
+; SI-NEXT: v_mov_b32_e32 v5, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[5:8], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[5:6], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v5
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v6
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB35_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v9, 1
+; SI-NEXT: v_readlane_b32 s6, v9, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v9, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -1826,29 +1826,27 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
; VI-NEXT: .LBB35_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v6, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB35_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_sub_i64_noret_offset_scalar:
@@ -1857,12 +1855,12 @@ define amdgpu_gfx void @global_atomic_sub_i64_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
; GFX9-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, s6, v2
-; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v1, vcc
+; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v3, v5, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1885,24 +1883,23 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
-; SI-NEXT: s_mov_b32 s34, s7
-; SI-NEXT: s_mov_b32 s35, s6
+; SI-NEXT: v_writelane_b32 v7, s6, 0
+; SI-NEXT: v_writelane_b32 v7, s7, 1
+; SI-NEXT: s_mov_b32 s35, s7
+; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
; SI-NEXT: .LBB36_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v2, vcc, s35, v4
-; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v0, vcc
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: v_subrev_i32_e32 v2, vcc, s34, v4
+; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v6, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -1918,10 +1915,10 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_cbranch_execnz .LBB36_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v7, 1
+; SI-NEXT: v_readlane_b32 s6, v7, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -1933,20 +1930,20 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB36_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v1, s7
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v0
+; VI-NEXT: v_mov_b32_e32 v8, v1
+; VI-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; VI-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB36_1
@@ -1960,18 +1957,18 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB36_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_subrev_co_u32_e32 v3, vcc, s6, v5
-; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v6, v0, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v6
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB36_1
@@ -1987,24 +1984,23 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
-; SI-NEXT: s_mov_b32 s34, s7
-; SI-NEXT: s_mov_b32 s35, s6
+; SI-NEXT: v_writelane_b32 v7, s6, 0
+; SI-NEXT: v_writelane_b32 v7, s7, 1
+; SI-NEXT: s_mov_b32 s35, s7
+; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
; SI-NEXT: .LBB37_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s34
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v2, vcc, s35, v4
-; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v0, vcc
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; SI-NEXT: v_subrev_i32_e32 v2, vcc, s34, v4
+; SI-NEXT: v_subb_u32_e32 v3, vcc, v5, v6, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -2020,10 +2016,10 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_cbranch_execnz .LBB37_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v7, 1
+; SI-NEXT: v_readlane_b32 s6, v7, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -2033,29 +2029,27 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
; VI-NEXT: .LBB37_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v1, s7
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_subb_u32_e32 v1, vcc, v3, v1, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v0
+; VI-NEXT: v_mov_b32_e32 v8, v1
+; VI-NEXT: v_subrev_u32_e32 v5, vcc, s6, v7
+; VI-NEXT: v_subb_u32_e32 v6, vcc, v8, v4, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[5:8] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB37_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_sub_i64_ret_offset_scalar:
@@ -2064,18 +2058,18 @@ define amdgpu_gfx i64 @global_atomic_sub_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB37_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_subrev_co_u32_e32 v3, vcc, s6, v5
-; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, v6, v0, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v6
+; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v7, v3, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[4:7], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB37_1
@@ -2569,13 +2563,13 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB44_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v1, s7, v3
; VI-NEXT: v_and_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -2666,28 +2660,26 @@ define amdgpu_gfx void @global_atomic_and_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB45_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v1, s7, v3
; VI-NEXT: v_and_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB45_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_and_i64_noret_offset_scalar:
@@ -2769,19 +2761,19 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB46_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_and_b32_e32 v1, s7, v3
-; VI-NEXT: v_and_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v5, s7, v7
+; VI-NEXT: v_and_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB46_1
@@ -2865,28 +2857,26 @@ define amdgpu_gfx i64 @global_atomic_and_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB47_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_and_b32_e32 v1, s7, v3
-; VI-NEXT: v_and_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v5, s7, v7
+; VI-NEXT: v_and_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB47_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_and_i64_ret_offset_scalar:
@@ -3423,13 +3413,13 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB54_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, s7, v3
; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: v_not_b32_e32 v1, v0
; VI-NEXT: v_not_b32_e32 v0, v6
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3526,17 +3516,15 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB55_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_and_b32_e32 v0, s7, v3
; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: v_not_b32_e32 v1, v0
; VI-NEXT: v_not_b32_e32 v0, v6
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -3544,12 +3532,12 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB55_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i64_noret_offset_scalar:
@@ -3635,21 +3623,21 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB56_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_and_b32_e32 v0, s7, v3
-; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_not_b32_e32 v1, v0
-; VI-NEXT: v_not_b32_e32 v0, v6
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v0, s7, v7
+; VI-NEXT: v_and_b32_e32 v1, s6, v6
+; VI-NEXT: v_not_b32_e32 v5, v0
+; VI-NEXT: v_not_b32_e32 v4, v1
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB56_1
@@ -3737,30 +3725,28 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB57_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_and_b32_e32 v0, s7, v3
-; VI-NEXT: v_and_b32_e32 v6, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_not_b32_e32 v1, v0
-; VI-NEXT: v_not_b32_e32 v0, v6
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_and_b32_e32 v0, s7, v7
+; VI-NEXT: v_and_b32_e32 v1, s6, v6
+; VI-NEXT: v_not_b32_e32 v5, v0
+; VI-NEXT: v_not_b32_e32 v4, v1
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB57_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_nand_i64_ret_offset_scalar:
@@ -4390,13 +4376,13 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB64_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_e32 v1, s7, v3
; VI-NEXT: v_or_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -4487,28 +4473,26 @@ define amdgpu_gfx void @global_atomic_or_i64_noret_offset_scalar(ptr addrspace(1
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB65_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_or_b32_e32 v1, s7, v3
; VI-NEXT: v_or_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB65_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_or_i64_noret_offset_scalar:
@@ -4590,19 +4574,19 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_scalar(ptr addrspace(1) inreg %p
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB66_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_or_b32_e32 v1, s7, v3
-; VI-NEXT: v_or_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_or_b32_e32 v5, s7, v7
+; VI-NEXT: v_or_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB66_1
@@ -4686,28 +4670,26 @@ define amdgpu_gfx i64 @global_atomic_or_i64_ret_offset_scalar(ptr addrspace(1) i
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB67_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_or_b32_e32 v1, s7, v3
-; VI-NEXT: v_or_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_or_b32_e32 v5, s7, v7
+; VI-NEXT: v_or_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB67_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_or_i64_ret_offset_scalar:
@@ -5220,13 +5202,13 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB74_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_xor_b32_e32 v1, s7, v3
; VI-NEXT: v_xor_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -5317,28 +5299,26 @@ define amdgpu_gfx void @global_atomic_xor_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB75_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_xor_b32_e32 v1, s7, v3
; VI-NEXT: v_xor_b32_e32 v0, s6, v2
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB75_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_xor_i64_noret_offset_scalar:
@@ -5420,19 +5400,19 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB76_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_xor_b32_e32 v1, s7, v3
-; VI-NEXT: v_xor_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_xor_b32_e32 v5, s7, v7
+; VI-NEXT: v_xor_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB76_1
@@ -5516,28 +5496,26 @@ define amdgpu_gfx i64 @global_atomic_xor_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB77_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_xor_b32_e32 v1, s7, v3
-; VI-NEXT: v_xor_b32_e32 v0, s6, v2
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_xor_b32_e32 v5, s7, v7
+; VI-NEXT: v_xor_b32_e32 v4, s6, v6
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB77_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_xor_i64_ret_offset_scalar:
@@ -6015,45 +5993,45 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB84_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB84_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -6065,16 +6043,16 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB84_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -6094,14 +6072,14 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB84_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -6123,45 +6101,45 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB85_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB85_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -6171,31 +6149,29 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB85_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB85_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i64_noret_offset_scalar:
@@ -6204,14 +6180,14 @@ define amdgpu_gfx void @global_atomic_max_i64_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB85_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -6234,26 +6210,26 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB86_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -6269,10 +6245,10 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_cbranch_execnz .LBB86_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -6284,22 +6260,22 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB86_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB86_1
@@ -6313,20 +6289,20 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB86_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB86_1
@@ -6342,26 +6318,26 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB87_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -6377,10 +6353,10 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_cbranch_execnz .LBB87_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -6390,31 +6366,29 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB87_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB87_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_max_i64_ret_offset_scalar:
@@ -6423,20 +6397,20 @@ define amdgpu_gfx i64 @global_atomic_max_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB87_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB87_1
@@ -6460,29 +6434,29 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB88_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s2
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB88_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6492,26 +6466,26 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB88_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -6528,24 +6502,24 @@ define amdgpu_kernel void @atomic_max_i64_addr64_offset(ptr addrspace(1) %out, i
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB88_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -6575,17 +6549,17 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB89_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -6611,68 +6585,68 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64_offset(ptr addrspace(1) %ou
; VI-LABEL: atomic_max_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB89_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB89_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB89_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB89_1
@@ -6701,29 +6675,29 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB90_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s2
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB90_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6735,30 +6709,30 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
-; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: s_add_u32 s4, s0, s4
+; VI-NEXT: s_addc_u32 s5, s1, s5
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
+; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB90_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB90_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
@@ -6767,24 +6741,24 @@ define amdgpu_kernel void @atomic_max_i64_addr64(ptr addrspace(1) %out, i64 %in,
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB90_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -6813,17 +6787,17 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB91_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -6851,64 +6825,64 @@ define amdgpu_kernel void @atomic_max_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB91_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB91_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB91_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_i64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB91_1
@@ -7371,45 +7345,45 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB98_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB98_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -7421,16 +7395,16 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB98_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -7450,14 +7424,14 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB98_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -7479,45 +7453,45 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB99_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB99_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -7527,31 +7501,29 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB99_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB99_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i64_noret_offset_scalar:
@@ -7560,14 +7532,14 @@ define amdgpu_gfx void @global_atomic_umax_i64_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB99_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -7590,26 +7562,26 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB100_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -7625,10 +7597,10 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_cbranch_execnz .LBB100_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -7640,22 +7612,22 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB100_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB100_1
@@ -7669,20 +7641,20 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB100_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB100_1
@@ -7698,26 +7670,26 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB101_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -7733,10 +7705,10 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_cbranch_execnz .LBB101_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -7746,31 +7718,29 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB101_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB101_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umax_i64_ret_offset_scalar:
@@ -7779,20 +7749,20 @@ define amdgpu_gfx i64 @global_atomic_umax_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB101_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB101_1
@@ -7816,29 +7786,29 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB102_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s2
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB102_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7848,26 +7818,26 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB102_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -7884,24 +7854,24 @@ define amdgpu_kernel void @atomic_umax_i64_addr64_offset(ptr addrspace(1) %out,
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB102_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -7931,17 +7901,17 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %o
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB103_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -7967,68 +7937,68 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64_offset(ptr addrspace(1) %o
; VI-LABEL: atomic_umax_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB103_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB103_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB103_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB103_1
@@ -8057,17 +8027,17 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB104_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -8095,64 +8065,64 @@ define amdgpu_kernel void @atomic_umax_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB104_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB104_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB104_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB104_1
@@ -8615,45 +8585,45 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB111_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB111_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -8665,16 +8635,16 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB111_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -8694,14 +8664,14 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_scalar(ptr addrspace(1) inr
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB111_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -8723,45 +8693,45 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB112_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB112_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -8771,31 +8741,29 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB112_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB112_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i64_noret_offset_scalar:
@@ -8804,14 +8772,14 @@ define amdgpu_gfx void @global_atomic_umin_i64_noret_offset_scalar(ptr addrspace
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB112_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -8834,26 +8802,26 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB113_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -8869,10 +8837,10 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; SI-NEXT: s_cbranch_execnz .LBB113_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -8884,22 +8852,22 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB113_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB113_1
@@ -8913,20 +8881,20 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_scalar(ptr addrspace(1) inreg
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB113_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB113_1
@@ -8942,26 +8910,26 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB114_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_u64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -8977,10 +8945,10 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_cbranch_execnz .LBB114_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -8990,31 +8958,29 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB114_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB114_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_umin_i64_ret_offset_scalar:
@@ -9023,20 +8989,20 @@ define amdgpu_gfx i64 @global_atomic_umin_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB114_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB114_1
@@ -9495,45 +9461,45 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB121_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB121_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -9545,16 +9511,16 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB121_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -9574,14 +9540,14 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_scalar(ptr addrspace(1) inre
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB121_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9603,45 +9569,45 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB122_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s35
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s34
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; SI-NEXT: s_cbranch_execnz .LBB122_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -9651,31 +9617,29 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB122_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB122_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i64_noret_offset_scalar:
@@ -9684,14 +9648,14 @@ define amdgpu_gfx void @global_atomic_min_i64_noret_offset_scalar(ptr addrspace(
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB122_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v5, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9714,26 +9678,26 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB123_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -9749,10 +9713,10 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; SI-NEXT: s_cbranch_execnz .LBB123_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -9764,22 +9728,22 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB123_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB123_1
@@ -9793,20 +9757,20 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_scalar(ptr addrspace(1) inreg %
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB123_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB123_1
@@ -9822,26 +9786,26 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v6, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v6, s6, 0
-; SI-NEXT: v_writelane_b32 v6, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[36:37], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB124_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[34:35], v[4:5]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s35
-; SI-NEXT: v_mov_b32_e32 v1, s34
-; SI-NEXT: v_cndmask_b32_e32 v3, v0, v5, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v1, v4, vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -9857,10 +9821,10 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; SI-NEXT: s_cbranch_execnz .LBB124_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[36:37]
-; SI-NEXT: v_readlane_b32 s7, v6, 1
-; SI-NEXT: v_readlane_b32 s6, v6, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v6, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -9870,31 +9834,29 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB124_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB124_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_min_i64_ret_offset_scalar:
@@ -9903,20 +9865,20 @@ define amdgpu_gfx i64 @global_atomic_min_i64_ret_offset_scalar(ptr addrspace(1)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[34:35], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB124_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v4, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[34:35]
; GFX9-NEXT: s_cbranch_execnz .LBB124_1
@@ -9940,29 +9902,29 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s8
; SI-NEXT: v_mov_b32_e32 v3, s9
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: .LBB125_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s2
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; SI-NEXT: s_cbranch_execnz .LBB125_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -9972,26 +9934,26 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
+; VI-NEXT: s_add_u32 s0, s0, s6
+; VI-NEXT: s_addc_u32 s1, s1, s7
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[4:5], 0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: .LBB125_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -10008,24 +9970,24 @@ define amdgpu_kernel void @atomic_min_i64_addr64_offset(ptr addrspace(1) %out, i
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 3
; GFX9-NEXT: s_add_u32 s0, s0, s4
; GFX9-NEXT: s_addc_u32 s1, s1, s5
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; GFX9-NEXT: s_mov_b64 s[4:5], 0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB125_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] offset:32 glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -10055,17 +10017,17 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %ou
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x8
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB126_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -10091,68 +10053,68 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64_offset(ptr addrspace(1) %ou
; VI-LABEL: atomic_min_i64_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[8:9], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
; VI-NEXT: s_add_u32 s0, s0, s6
; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x20
+; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x20
; VI-NEXT: s_add_u32 s0, s0, 32
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s6
+; VI-NEXT: v_mov_b32_e32 v3, s7
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: .LBB126_1: ; %atomicrmw.start
-; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: ; =>This Inner Loop Header: Depth=1
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; VI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; VI-NEXT: s_cbranch_execnz .LBB126_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[8:9]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x20
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB126_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] offset:32 glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB126_1
@@ -10178,6 +10140,8 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_load_dwordx2 s[10:11], s[0:1], 0x0
+; SI-NEXT: v_mov_b32_e32 v4, s3
+; SI-NEXT: v_mov_b32_e32 v5, s2
; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt lgkmcnt(0)
@@ -10186,22 +10150,20 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; SI-NEXT: .LBB127_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s2
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SI-NEXT: s_cbranch_execnz .LBB127_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -10213,18 +10175,18 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
+; VI-NEXT: v_mov_b32_e32 v6, s3
+; VI-NEXT: v_mov_b32_e32 v7, s2
+; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v3, s7
; VI-NEXT: .LBB127_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
+; VI-NEXT: v_cndmask_b32_e32 v1, v6, v3, vcc
+; VI-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -10241,20 +10203,20 @@ define amdgpu_kernel void @atomic_min_i64(ptr addrspace(1) %out, i64 %in) {
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[4:5], 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, s2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: .LBB127_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[2:3], v[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v2, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[0:1] glc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -10282,17 +10244,17 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr
; SI-NEXT: s_addc_u32 s9, s1, s7
; SI-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v8, s5
+; SI-NEXT: v_mov_b32_e32 v9, s4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: v_mov_b32_e32 v3, s7
; SI-NEXT: .LBB128_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v0, s5
+; SI-NEXT: v_cndmask_b32_e32 v1, v8, v3, vcc
+; SI-NEXT: v_cndmask_b32_e32 v0, v9, v2, vcc
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s4
-; SI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; SI-NEXT: v_cndmask_b32_e32 v0, v4, v2, vcc
; SI-NEXT: v_mov_b32_e32 v7, v3
; SI-NEXT: v_mov_b32_e32 v6, v2
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -10320,64 +10282,64 @@ define amdgpu_kernel void @atomic_min_i64_ret_addr64(ptr addrspace(1) %out, ptr
; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], 3
-; VI-NEXT: s_add_u32 s0, s0, s6
-; VI-NEXT: s_addc_u32 s1, s1, s7
-; VI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x0
-; VI-NEXT: s_mov_b64 s[6:7], 0
+; VI-NEXT: s_add_u32 s6, s0, s6
+; VI-NEXT: s_addc_u32 s7, s1, s7
+; VI-NEXT: s_load_dwordx2 s[8:9], s[6:7], 0x0
+; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v4, s5
+; VI-NEXT: v_mov_b32_e32 v5, s4
+; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v2, s8
+; VI-NEXT: v_mov_b32_e32 v3, s9
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: .LBB128_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: v_mov_b32_e32 v6, s4
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: v_cndmask_b32_e32 v1, v0, v3, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v9, v3
+; VI-NEXT: v_mov_b32_e32 v8, v2
+; VI-NEXT: v_cmp_ge_i64_e32 vcc, s[4:5], v[8:9]
+; VI-NEXT: v_cndmask_b32_e32 v7, v4, v9, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v5, v8, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[2:3], v[0:1], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; VI-NEXT: s_andn2_b64 exec, exec, s[6:7]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[2:3], v[8:9]
+; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
; VI-NEXT: s_cbranch_execnz .LBB128_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b64 exec, exec, s[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i64_ret_addr64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b64 s[0:1], s[14:15], 3
; GFX9-NEXT: s_add_u32 s0, s8, s0
; GFX9-NEXT: s_addc_u32 s1, s9, s1
; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v2, s13
+; GFX9-NEXT: v_mov_b32_e32 v3, s12
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: .LBB128_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[5:6]
-; GFX9-NEXT: v_mov_b32_e32 v0, s13
-; GFX9-NEXT: v_mov_b32_e32 v1, s12
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v0, v6, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v1, v5, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[0:1] glc
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_ge_i64_e32 vcc, s[12:13], v[7:8]
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v2, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v7, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[5:8], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX9-NEXT: s_cbranch_execnz .LBB128_1
@@ -10912,14 +10874,14 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_scalar(ptr addrspace(1
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: .LBB135_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11018,18 +10980,16 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_offset_scalar(ptr addr
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB136_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11037,12 +10997,12 @@ define amdgpu_gfx void @global_atomic_uinc_wrap_i64_noret_offset_scalar(ptr addr
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB136_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_uinc_wrap_i64_noret_offset_scalar:
@@ -11130,22 +11090,22 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_scalar(ptr addrspace(1) i
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[34:35], 0
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB137_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; VI-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; VI-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB137_1
@@ -11235,31 +11195,29 @@ define amdgpu_gfx i64 @global_atomic_uinc_wrap_i64_ret_offset_scalar(ptr addrspa
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; VI-NEXT: s_add_u32 s34, s4, 32
; VI-NEXT: s_addc_u32 s35, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s34
-; VI-NEXT: v_mov_b32_e32 v1, s35
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[34:35], 0
; VI-NEXT: .LBB138_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_mov_b32_e32 v4, s34
-; VI-NEXT: v_mov_b32_e32 v5, s35
-; VI-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_mov_b32_e32 v6, v0
+; VI-NEXT: v_mov_b32_e32 v7, v1
+; VI-NEXT: v_add_u32_e32 v0, vcc, 1, v6
+; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v7, vcc
+; VI-NEXT: v_cmp_gt_u64_e32 vcc, s[6:7], v[6:7]
+; VI-NEXT: v_cndmask_b32_e32 v5, 0, v1, vcc
+; VI-NEXT: v_cndmask_b32_e32 v4, 0, v0, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[4:7] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
-; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[6:7]
+; VI-NEXT: s_or_b64 s[34:35], vcc, s[34:35]
+; VI-NEXT: s_andn2_b64 exec, exec, s[34:35]
; VI-NEXT: s_cbranch_execnz .LBB138_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[36:37]
+; VI-NEXT: s_or_b64 exec, exec, s[34:35]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_uinc_wrap_i64_ret_offset_scalar:
@@ -11776,48 +11734,48 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB145_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]
-; SI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s35
-; SI-NEXT: v_mov_b32_e32 v5, s34
-; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]
+; SI-NEXT: v_subrev_i32_e64 v0, s[36:37], 1, v2
+; SI-NEXT: v_subbrev_u32_e64 v1, s[36:37], 0, v3, s[36:37]
+; SI-NEXT: s_or_b64 vcc, s[36:37], vcc
; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 glc
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB145_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -11829,17 +11787,17 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_mov_b64 s[36:37], 0
-; VI-NEXT: .LBB145_1: ; %atomicrmw.start
-; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; VI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
; VI-NEXT: v_mov_b32_e32 v6, s7
; VI-NEXT: v_mov_b32_e32 v7, s6
-; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
; VI-NEXT: v_mov_b32_e32 v4, s4
; VI-NEXT: v_mov_b32_e32 v5, s5
+; VI-NEXT: .LBB145_1: ; %atomicrmw.start
+; VI-NEXT: ; =>This Inner Loop Header: Depth=1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; VI-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; VI-NEXT: s_or_b64 vcc, s[34:35], vcc
; VI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11861,15 +11819,15 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_scalar(ptr addrspace(1
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB145_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX9-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] glc
@@ -11893,48 +11851,48 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v10, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v8, s6, 0
-; SI-NEXT: v_writelane_b32 v8, s7, 1
+; SI-NEXT: v_writelane_b32 v10, s6, 0
+; SI-NEXT: v_writelane_b32 v10, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[2:3], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v4, s35
+; SI-NEXT: v_mov_b32_e32 v5, s34
; SI-NEXT: .LBB146_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v2
-; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[2:3]
-; SI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v4, s35
-; SI-NEXT: v_mov_b32_e32 v5, s34
-; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
+; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[2:3]
+; SI-NEXT: v_subrev_i32_e64 v0, s[36:37], 1, v2
+; SI-NEXT: v_subbrev_u32_e64 v1, s[36:37], 0, v3, s[36:37]
+; SI-NEXT: s_or_b64 vcc, s[36:37], vcc
; SI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
-; SI-NEXT: v_mov_b32_e32 v7, v3
-; SI-NEXT: v_mov_b32_e32 v6, v2
-; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v4, v0
-; SI-NEXT: buffer_atomic_cmpswap_x2 v[4:7], off, s[4:7], 0 offset:32 glc
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_mov_b32_e32 v9, v3
+; SI-NEXT: v_mov_b32_e32 v8, v2
+; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_mov_b32_e32 v6, v0
+; SI-NEXT: buffer_atomic_cmpswap_x2 v[6:9], off, s[4:7], 0 offset:32 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
-; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[4:5], v[2:3]
-; SI-NEXT: v_mov_b32_e32 v2, v4
+; SI-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[2:3]
+; SI-NEXT: v_mov_b32_e32 v2, v6
; SI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; SI-NEXT: v_mov_b32_e32 v3, v5
+; SI-NEXT: v_mov_b32_e32 v3, v7
; SI-NEXT: s_andn2_b64 exec, exec, s[38:39]
; SI-NEXT: s_cbranch_execnz .LBB146_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v8, 1
-; SI-NEXT: v_readlane_b32 s6, v8, 0
+; SI-NEXT: v_readlane_b32 s7, v10, 1
+; SI-NEXT: v_readlane_b32 s6, v10, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v10, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -11942,23 +11900,21 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
; VI-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: s_add_u32 s36, s4, 32
-; VI-NEXT: s_addc_u32 s37, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s36
-; VI-NEXT: v_mov_b32_e32 v1, s37
-; VI-NEXT: flat_load_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_mov_b64 s[38:39], 0
+; VI-NEXT: s_add_u32 s34, s4, 32
+; VI-NEXT: s_addc_u32 s35, s5, 0
+; VI-NEXT: v_mov_b32_e32 v4, s34
+; VI-NEXT: v_mov_b32_e32 v5, s35
+; VI-NEXT: flat_load_dwordx2 v[2:3], v[4:5]
+; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v6, s7
+; VI-NEXT: v_mov_b32_e32 v7, s6
; VI-NEXT: .LBB146_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, 1, v2
-; VI-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; VI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v3, vcc
-; VI-NEXT: v_mov_b32_e32 v6, s7
-; VI-NEXT: v_mov_b32_e32 v7, s6
-; VI-NEXT: s_or_b64 vcc, vcc, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v4, s36
-; VI-NEXT: v_mov_b32_e32 v5, s37
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; VI-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v2
+; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; VI-NEXT: s_or_b64 vcc, s[34:35], vcc
; VI-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
@@ -11966,12 +11922,12 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
+; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB146_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[38:39]
+; VI-NEXT: s_or_b64 exec, exec, s[36:37]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_udec_wrap_i64_noret_offset_scalar:
@@ -11980,15 +11936,15 @@ define amdgpu_gfx void @global_atomic_udec_wrap_i64_noret_offset_scalar(ptr addr
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v5, s7
+; GFX9-NEXT: v_mov_b32_e32 v6, s6
; GFX9-NEXT: .LBB146_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_subrev_co_u32_e32 v0, vcc, 1, v2
-; GFX9-NEXT: v_cmp_lt_u64_e64 s[34:35], s[6:7], v[2:3]
-; GFX9-NEXT: v_subbrev_co_u32_e32 v1, vcc, 0, v3, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s7
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: s_or_b64 vcc, vcc, s[34:35]
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v2
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v3, s[34:35]
+; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v4, v[0:3], s[4:5] offset:32 glc
@@ -12013,28 +11969,29 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v7, s6, 0
-; SI-NEXT: v_writelane_b32 v7, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB147_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v4
-; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[4:5]
-; SI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v5, vcc
-; SI-NEXT: v_mov_b32_e32 v2, s35
-; SI-NEXT: v_mov_b32_e32 v6, s34
-; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
-; SI-NEXT: v_cndmask_b32_e32 v3, v1, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v0, v6, vcc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_subrev_i32_e64 v0, s[36:37], 1, v4
+; SI-NEXT: v_subbrev_u32_e64 v1, s[36:37], 0, v5, s[36:37]
+; SI-NEXT: s_or_b64 vcc, s[36:37], vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v1, v6, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v0, v7, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -12050,10 +12007,10 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
; SI-NEXT: s_cbranch_execnz .LBB147_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v7, 1
-; SI-NEXT: v_readlane_b32 s6, v7, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -12065,25 +12022,25 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: .LBB147_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; VI-NEXT: s_or_b64 vcc, s[34:35], vcc
-; VI-NEXT: v_mov_b32_e32 v4, s4
-; VI-NEXT: v_mov_b32_e32 v5, s5
-; VI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB147_1
@@ -12097,23 +12054,23 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_scalar(ptr addrspace(1) i
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5]
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB147_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_subrev_co_u32_e64 v3, s[34:35], 1, v5
-; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[34:35], 0, v6, s[34:35]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v7
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v8, s[34:35]
; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] glc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB147_1
@@ -12129,28 +12086,29 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_store_dword v7, off, s[0:3], s32 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v8, off, s[0:3], s32 ; 4-byte Folded Spill
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_writelane_b32 v7, s6, 0
-; SI-NEXT: v_writelane_b32 v7, s7, 1
+; SI-NEXT: v_writelane_b32 v8, s6, 0
+; SI-NEXT: v_writelane_b32 v8, s7, 1
; SI-NEXT: s_mov_b32 s35, s7
; SI-NEXT: s_mov_b32 s34, s6
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: buffer_load_dwordx2 v[4:5], off, s[4:7], 0 offset:32
; SI-NEXT: s_mov_b64 s[38:39], 0
+; SI-NEXT: v_mov_b32_e32 v6, s35
+; SI-NEXT: v_mov_b32_e32 v7, s34
; SI-NEXT: .LBB148_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v0, vcc, 1, v4
-; SI-NEXT: v_cmp_lt_u64_e64 s[36:37], s[34:35], v[4:5]
-; SI-NEXT: v_subbrev_u32_e32 v1, vcc, 0, v5, vcc
-; SI-NEXT: v_mov_b32_e32 v2, s35
-; SI-NEXT: v_mov_b32_e32 v6, s34
-; SI-NEXT: s_or_b64 vcc, vcc, s[36:37]
-; SI-NEXT: v_cndmask_b32_e32 v3, v1, v2, vcc
-; SI-NEXT: v_cndmask_b32_e32 v2, v0, v6, vcc
+; SI-NEXT: s_waitcnt vmcnt(0)
+; SI-NEXT: v_cmp_lt_u64_e32 vcc, s[34:35], v[4:5]
+; SI-NEXT: s_waitcnt expcnt(0)
+; SI-NEXT: v_subrev_i32_e64 v0, s[36:37], 1, v4
+; SI-NEXT: v_subbrev_u32_e64 v1, s[36:37], 0, v5, s[36:37]
+; SI-NEXT: s_or_b64 vcc, s[36:37], vcc
+; SI-NEXT: v_cndmask_b32_e32 v3, v1, v6, vcc
+; SI-NEXT: v_cndmask_b32_e32 v2, v0, v7, vcc
; SI-NEXT: v_mov_b32_e32 v0, v2
; SI-NEXT: v_mov_b32_e32 v1, v3
; SI-NEXT: v_mov_b32_e32 v2, v4
@@ -12166,10 +12124,10 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
; SI-NEXT: s_cbranch_execnz .LBB148_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_or_b64 exec, exec, s[38:39]
-; SI-NEXT: v_readlane_b32 s7, v7, 1
-; SI-NEXT: v_readlane_b32 s6, v7, 0
+; SI-NEXT: v_readlane_b32 s7, v8, 1
+; SI-NEXT: v_readlane_b32 s6, v8, 0
; SI-NEXT: s_xor_saveexec_b64 s[34:35], -1
-; SI-NEXT: buffer_load_dword v7, off, s[0:3], s32 ; 4-byte Folded Reload
+; SI-NEXT: buffer_load_dword v8, off, s[0:3], s32 ; 4-byte Folded Reload
; SI-NEXT: s_mov_b64 exec, s[34:35]
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
@@ -12177,36 +12135,34 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
; VI-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:
; VI: ; %bb.0:
; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: s_add_u32 s36, s4, 32
-; VI-NEXT: s_addc_u32 s37, s5, 0
-; VI-NEXT: v_mov_b32_e32 v0, s36
-; VI-NEXT: v_mov_b32_e32 v1, s37
-; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_mov_b64 s[38:39], 0
+; VI-NEXT: s_add_u32 s34, s4, 32
+; VI-NEXT: s_addc_u32 s35, s5, 0
+; VI-NEXT: v_mov_b32_e32 v2, s34
+; VI-NEXT: v_mov_b32_e32 v3, s35
+; VI-NEXT: flat_load_dwordx2 v[0:1], v[2:3]
+; VI-NEXT: s_mov_b64 s[36:37], 0
+; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v5, s6
; VI-NEXT: .LBB148_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, v1
-; VI-NEXT: v_mov_b32_e32 v2, v0
-; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
-; VI-NEXT: v_subrev_u32_e64 v7, s[34:35], 1, v2
-; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v3, s[34:35]
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: v_mov_b32_e32 v6, s6
+; VI-NEXT: v_mov_b32_e32 v9, v1
+; VI-NEXT: v_mov_b32_e32 v8, v0
+; VI-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
+; VI-NEXT: v_subrev_u32_e64 v0, s[34:35], 1, v8
+; VI-NEXT: v_subbrev_u32_e64 v1, s[34:35], 0, v9, s[34:35]
; VI-NEXT: s_or_b64 vcc, s[34:35], vcc
-; VI-NEXT: v_mov_b32_e32 v4, s36
-; VI-NEXT: v_mov_b32_e32 v5, s37
-; VI-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; VI-NEXT: v_cndmask_b32_e32 v0, v7, v6, vcc
-; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[4:5], v[0:3] glc
+; VI-NEXT: v_cndmask_b32_e32 v7, v1, v4, vcc
+; VI-NEXT: v_cndmask_b32_e32 v6, v0, v5, vcc
+; VI-NEXT: flat_atomic_cmpswap_x2 v[0:1], v[2:3], v[6:9] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
-; VI-NEXT: s_or_b64 s[38:39], vcc, s[38:39]
-; VI-NEXT: s_andn2_b64 exec, exec, s[38:39]
+; VI-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[8:9]
+; VI-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
+; VI-NEXT: s_andn2_b64 exec, exec, s[36:37]
; VI-NEXT: s_cbranch_execnz .LBB148_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
-; VI-NEXT: s_or_b64 exec, exec, s[38:39]
+; VI-NEXT: s_or_b64 exec, exec, s[36:37]
; VI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: global_atomic_udec_wrap_i64_ret_offset_scalar:
@@ -12215,23 +12171,23 @@ define amdgpu_gfx i64 @global_atomic_udec_wrap_i64_ret_offset_scalar(ptr addrspa
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[4:5] offset:32
; GFX9-NEXT: s_mov_b64 s[36:37], 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
+; GFX9-NEXT: v_mov_b32_e32 v4, s6
; GFX9-NEXT: .LBB148_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
-; GFX9-NEXT: v_subrev_co_u32_e64 v3, s[34:35], 1, v5
-; GFX9-NEXT: v_subbrev_co_u32_e64 v4, s[34:35], 0, v6, s[34:35]
-; GFX9-NEXT: v_mov_b32_e32 v0, s7
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, v0
+; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[7:8]
+; GFX9-NEXT: v_subrev_co_u32_e64 v0, s[34:35], 1, v7
+; GFX9-NEXT: v_subbrev_co_u32_e64 v1, s[34:35], 0, v8, s[34:35]
; GFX9-NEXT: s_or_b64 vcc, s[34:35], vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc
-; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[3:6], s[4:5] offset:32 glc
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v4, vcc
+; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v2, v[5:8], s[4:5] offset:32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[5:6]
+; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[7:8]
; GFX9-NEXT: s_or_b64 s[36:37], vcc, s[36:37]
; GFX9-NEXT: s_andn2_b64 exec, exec, s[36:37]
; GFX9-NEXT: s_cbranch_execnz .LBB148_1
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
index 660a281f95824..c1e5a18ec6aa2 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmax.ll
@@ -352,14 +352,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -693,14 +693,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1331,14 +1331,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1672,14 +1672,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2310,14 +2310,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_max_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_max_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2651,14 +2651,14 @@ define amdgpu_kernel void @global_atomic_fmax_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_max_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -3367,15 +3367,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3552,16 +3552,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3621,15 +3621,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -4029,16 +4030,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -4122,15 +4123,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
@@ -4562,15 +4563,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4747,16 +4748,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4816,15 +4817,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -5224,16 +5226,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -5317,15 +5319,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
@@ -5757,15 +5759,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -5942,16 +5944,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -6011,15 +6013,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_max_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -6419,16 +6422,16 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -6512,15 +6515,15 @@ define amdgpu_kernel void @global_atomic_fmax_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
index 9c2452ca034e7..85bcfd88181b9 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_scan_fmin.ll
@@ -352,14 +352,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB1_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -693,14 +693,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_agent_scope_
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB1_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1331,14 +1331,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB3_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -1672,14 +1672,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_one_as_scope
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB3_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2310,14 +2310,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: v_max_f32_e32 v2, v2, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v3, s[0:1]
; GFX9-NEXT: .LBB5_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f32_e32 v0, v2, v2
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f32_e32 v4, v1, v1
-; GFX9-NEXT: v_min_f32_e32 v0, v4, v0
+; GFX9-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-NEXT: v_min_f32_e32 v0, v0, v2
; GFX9-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -2651,14 +2651,14 @@ define amdgpu_kernel void @global_atomic_fmin_uni_address_div_value_default_scop
; GFX9-DPP-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-DPP-NEXT: v_mov_b32_e32 v2, 0
; GFX9-DPP-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-DPP-NEXT: v_max_f32_e64 v6, s4, s4
; GFX9-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DPP-NEXT: global_load_dword v1, v2, s[0:1]
; GFX9-DPP-NEXT: .LBB5_2: ; %atomicrmw.start
; GFX9-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-DPP-NEXT: v_max_f32_e64 v0, s4, s4
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DPP-NEXT: v_max_f32_e32 v6, v1, v1
-; GFX9-DPP-NEXT: v_min_f32_e32 v0, v6, v0
+; GFX9-DPP-NEXT: v_max_f32_e32 v0, v1, v1
+; GFX9-DPP-NEXT: v_min_f32_e32 v0, v0, v6
; GFX9-DPP-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX9-DPP-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
@@ -3367,15 +3367,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3552,16 +3552,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -3621,15 +3621,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB7_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -4029,16 +4030,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -4122,15 +4123,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_agent
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB7_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB7_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
@@ -4562,15 +4563,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4747,16 +4748,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -4816,15 +4817,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB9_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -5224,16 +5226,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -5317,15 +5319,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_one_a
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB9_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB9_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
@@ -5757,15 +5759,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX9-NEXT: ; %bb.3:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[34:35], 0x24
; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX9-NEXT: s_mov_b64 s[2:3], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[2:3], v6, s[0:1]
; GFX9-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
-; GFX9-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX9-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
+; GFX9-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX9-NEXT: global_atomic_cmpswap_x2 v[0:1], v6, v[0:3], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -5942,16 +5944,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-NEXT: ; %bb.3:
; GFX1164-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1164-NEXT: v_mov_b32_e32 v6, 0
+; GFX1164-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1164-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1164-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1164-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1164-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1164-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1164-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1164-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1164-NEXT: s_waitcnt vmcnt(0)
; GFX1164-NEXT: v_cmp_eq_u64_e32 vcc, v[0:1], v[2:3]
@@ -6011,15 +6013,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-NEXT: ; %bb.3:
; GFX1132-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
; GFX1132-NEXT: v_mov_b32_e32 v6, 0
+; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1132-NEXT: v_max_f64 v[4:5], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: global_load_b64 v[2:3], v6, s[0:1]
; GFX1132-NEXT: .LBB11_4: ; %atomicrmw.start
; GFX1132-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-NEXT: v_max_f64 v[0:1], v[4:5], v[4:5]
; GFX1132-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-NEXT: v_max_f64 v[7:8], v[2:3], v[2:3]
+; GFX1132-NEXT: v_max_f64 v[0:1], v[2:3], v[2:3]
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-NEXT: v_min_f64 v[0:1], v[7:8], v[0:1]
+; GFX1132-NEXT: v_min_f64 v[0:1], v[0:1], v[4:5]
; GFX1132-NEXT: global_atomic_cmpswap_b64 v[0:1], v6, v[0:3], s[0:1] glc
; GFX1132-NEXT: s_waitcnt vmcnt(0)
; GFX1132-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
@@ -6419,16 +6422,16 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1164-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1164-DPP-NEXT: ; %bb.1:
; GFX1164-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1164-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_mov_b64 s[2:3], 0
; GFX1164-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1164-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1164-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1164-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1164-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1164-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1164-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1164-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1164-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1164-DPP-NEXT: v_cmp_eq_u64_e32 vcc, v[6:7], v[8:9]
@@ -6512,15 +6515,15 @@ define amdgpu_kernel void @global_atomic_fmin_double_uni_address_div_value_defau
; GFX1132-DPP-NEXT: s_cbranch_execz .LBB11_3
; GFX1132-DPP-NEXT: ; %bb.1:
; GFX1132-DPP-NEXT: s_load_b64 s[0:1], s[34:35], 0x24
+; GFX1132-DPP-NEXT: v_max_f64 v[0:1], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-DPP-NEXT: global_load_b64 v[8:9], v10, s[0:1]
; GFX1132-DPP-NEXT: .LBB11_2: ; %atomicrmw.start
; GFX1132-DPP-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[0:1], v[0:1]
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
-; GFX1132-DPP-NEXT: v_max_f64 v[11:12], v[8:9], v[8:9]
+; GFX1132-DPP-NEXT: v_max_f64 v[6:7], v[8:9], v[8:9]
; GFX1132-DPP-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[11:12], v[6:7]
+; GFX1132-DPP-NEXT: v_min_f64 v[6:7], v[6:7], v[0:1]
; GFX1132-DPP-NEXT: global_atomic_cmpswap_b64 v[6:7], v10, v[6:9], s[0:1] glc
; GFX1132-DPP-NEXT: s_waitcnt vmcnt(0)
; GFX1132-DPP-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[6:7], v[8:9]
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-call.ll b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
index 8951b179ce299..acc3a8598557d 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-call.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-call.ll
@@ -932,15 +932,17 @@ define void @test_indirect_call_vgpr_ptr_inreg_arg(ptr %fptr) #0 {
; GISEL-NEXT: v_writelane_b32 v40, s30, 14
; GISEL-NEXT: v_writelane_b32 v40, s31, 15
; GISEL-NEXT: s_mov_b64 s[6:7], exec
+; GISEL-NEXT: v_mov_b32_e32 v2, 0x7b
; GISEL-NEXT: .LBB6_1: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_readfirstlane_b32 s8, v0
-; GISEL-NEXT: v_readfirstlane_b32 s9, v1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[8:9], v[0:1]
-; GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc
-; GISEL-NEXT: s_movk_i32 s4, 0x7b
-; GISEL-NEXT: s_swappc_b64 s[30:31], s[8:9]
+; GISEL-NEXT: v_readfirstlane_b32 s10, v0
+; GISEL-NEXT: v_readfirstlane_b32 s11, v1
+; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, s[10:11], v[0:1]
+; GISEL-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GISEL-NEXT: s_swappc_b64 s[30:31], s[10:11]
; GISEL-NEXT: ; implicit-def: $vgpr0
-; GISEL-NEXT: s_xor_b64 exec, exec, s[10:11]
+; GISEL-NEXT: ; implicit-def: $vgpr2
+; GISEL-NEXT: s_xor_b64 exec, exec, s[8:9]
; GISEL-NEXT: s_cbranch_execnz .LBB6_1
; GISEL-NEXT: ; %bb.2:
; GISEL-NEXT: s_mov_b64 exec, s[6:7]
diff --git a/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll b/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
index cfaae36ee798f..52ec0a6246373 100644
--- a/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue130120-eliminate-frame-index.ll
@@ -5,58 +5,58 @@ define amdgpu_gfx [13 x i32] @issue130120() {
; CHECK-LABEL: issue130120:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: s_movk_i32 s0, 0xf4
-; CHECK-NEXT: s_movk_i32 s1, 0xf8
-; CHECK-NEXT: s_add_i32 s34, s32, s0
-; CHECK-NEXT: s_add_i32 s35, s32, s1
-; CHECK-NEXT: s_movk_i32 s0, 0xfc
-; CHECK-NEXT: s_movk_i32 s1, 0x100
-; CHECK-NEXT: s_add_i32 s36, s32, s0
-; CHECK-NEXT: s_add_i32 s37, s32, s1
-; CHECK-NEXT: s_movk_i32 s0, 0x104
-; CHECK-NEXT: s_movk_i32 s1, 0x108
; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: s_add_i32 s38, s32, s0
-; CHECK-NEXT: s_add_i32 s39, s32, s1
-; CHECK-NEXT: s_movk_i32 s0, 0x110
-; CHECK-NEXT: s_movk_i32 s1, 0x120
-; CHECK-NEXT: s_add_i32 s3, s32, 0xf0
-; CHECK-NEXT: s_add_i32 s40, s32, s0
-; CHECK-NEXT: s_add_i32 s41, s32, s1
-; CHECK-NEXT: s_or_b32 s42, s32, 4
-; CHECK-NEXT: s_or_b32 s43, s32, 8
-; CHECK-NEXT: s_or_b32 s44, s32, 12
-; CHECK-NEXT: s_add_i32 s45, s32, 16
-; CHECK-NEXT: s_add_i32 s46, s32, 20
-; CHECK-NEXT: s_add_i32 s47, s32, 24
-; CHECK-NEXT: s_mov_b32 s49, 1
-; CHECK-NEXT: s_movk_i32 s48, 0x990
-; CHECK-NEXT: s_mov_b32 s0, 0
+; CHECK-NEXT: s_movk_i32 s1, 0xf4
+; CHECK-NEXT: s_movk_i32 s2, 0xf8
+; CHECK-NEXT: s_movk_i32 s3, 0xfc
+; CHECK-NEXT: s_movk_i32 s34, 0x100
+; CHECK-NEXT: v_mov_b32_e32 v1, v0
+; CHECK-NEXT: s_movk_i32 s35, 0x104
+; CHECK-NEXT: s_movk_i32 s36, 0x108
+; CHECK-NEXT: s_movk_i32 s37, 0x110
+; CHECK-NEXT: s_movk_i32 s38, 0x120
+; CHECK-NEXT: s_add_i32 s0, s32, 0xf0
+; CHECK-NEXT: s_add_i32 s1, s32, s1
+; CHECK-NEXT: s_add_i32 s2, s32, s2
+; CHECK-NEXT: s_add_i32 s3, s32, s3
+; CHECK-NEXT: s_add_i32 s34, s32, s34
+; CHECK-NEXT: s_add_i32 s35, s32, s35
+; CHECK-NEXT: s_add_i32 s36, s32, s36
+; CHECK-NEXT: s_add_i32 s37, s32, s37
+; CHECK-NEXT: s_add_i32 s38, s32, s38
+; CHECK-NEXT: s_or_b32 s39, s32, 4
+; CHECK-NEXT: s_or_b32 s40, s32, 8
+; CHECK-NEXT: s_or_b32 s41, s32, 12
+; CHECK-NEXT: s_add_i32 s42, s32, 16
+; CHECK-NEXT: s_add_i32 s43, s32, 20
+; CHECK-NEXT: s_add_i32 s44, s32, 24
+; CHECK-NEXT: s_mov_b32 s46, 1
+; CHECK-NEXT: s_movk_i32 s45, 0x990
+; CHECK-NEXT: s_mov_b32 s48, 0
; CHECK-NEXT: .LBB0_1: ; %bb3
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_cmp_eq_u32 s49, 0
-; CHECK-NEXT: s_mov_b32 s1, s0
-; CHECK-NEXT: s_mov_b32 s2, s0
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v4, s2
-; CHECK-NEXT: s_cselect_b32 s51, 0, s34
-; CHECK-NEXT: s_cselect_b32 s55, 0, s38
-; CHECK-NEXT: s_cselect_b32 s52, 0, s35
-; CHECK-NEXT: s_cselect_b32 s56, 0, s39
-; CHECK-NEXT: s_cselect_b32 vcc_lo, 0, s46
-; CHECK-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; CHECK-NEXT: s_cselect_b32 s50, s48, 0xf0
-; CHECK-NEXT: s_cselect_b32 s53, 0, s36
-; CHECK-NEXT: s_cselect_b32 s54, 0, s37
-; CHECK-NEXT: s_cselect_b32 s57, 0, s40
-; CHECK-NEXT: s_cselect_b32 s58, 0, s41
-; CHECK-NEXT: s_cselect_b32 s59, 0, s3
-; CHECK-NEXT: s_cselect_b32 s60, 0, s42
-; CHECK-NEXT: s_cselect_b32 s61, 0, s43
-; CHECK-NEXT: s_cselect_b32 s62, 0, s44
-; CHECK-NEXT: s_cselect_b32 s63, 0, s45
-; CHECK-NEXT: s_cselect_b32 vcc_hi, 0, s47
-; CHECK-NEXT: s_mov_b32 s49, s0
+; CHECK-NEXT: s_cmp_eq_u32 s46, 0
+; CHECK-NEXT: s_mov_b32 s49, s48
+; CHECK-NEXT: s_mov_b32 s50, s48
+; CHECK-NEXT: s_cselect_b32 s51, 0, s1
+; CHECK-NEXT: s_cselect_b32 s55, 0, s35
+; CHECK-NEXT: s_cselect_b32 s52, 0, s2
+; CHECK-NEXT: s_cselect_b32 s56, 0, s36
+; CHECK-NEXT: s_cselect_b32 vcc_lo, 0, s43
+; CHECK-NEXT: v_dual_mov_b32 v2, s48 :: v_dual_mov_b32 v3, s49
+; CHECK-NEXT: v_mov_b32_e32 v4, s50
+; CHECK-NEXT: s_cselect_b32 s47, s45, 0xf0
+; CHECK-NEXT: s_cselect_b32 s53, 0, s3
+; CHECK-NEXT: s_cselect_b32 s54, 0, s34
+; CHECK-NEXT: s_cselect_b32 s57, 0, s37
+; CHECK-NEXT: s_cselect_b32 s58, 0, s38
+; CHECK-NEXT: s_cselect_b32 s59, 0, s0
+; CHECK-NEXT: s_cselect_b32 s60, 0, s39
+; CHECK-NEXT: s_cselect_b32 s61, 0, s40
+; CHECK-NEXT: s_cselect_b32 s62, 0, s41
+; CHECK-NEXT: s_cselect_b32 s63, 0, s42
+; CHECK-NEXT: s_cselect_b32 vcc_hi, 0, s44
+; CHECK-NEXT: s_mov_b32 s46, s48
; CHECK-NEXT: scratch_store_b32 off, v0, s51
; CHECK-NEXT: scratch_store_b32 off, v0, s52
; CHECK-NEXT: scratch_store_b32 off, v0, s53
@@ -64,7 +64,7 @@ define amdgpu_gfx [13 x i32] @issue130120() {
; CHECK-NEXT: scratch_store_b32 off, v0, s55
; CHECK-NEXT: scratch_store_b64 off, v[0:1], s56
; CHECK-NEXT: scratch_store_b32 off, v0, s57
-; CHECK-NEXT: scratch_store_b32 off, v0, s50
+; CHECK-NEXT: scratch_store_b32 off, v0, s47
; CHECK-NEXT: scratch_store_b96 off, v[2:4], s58
; CHECK-NEXT: scratch_store_b96 off, v[2:4], s59
; CHECK-NEXT: scratch_store_b32 off, v0, s60
diff --git a/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll b/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
index 442bcb1990bbd..a0d4945eb3975 100644
--- a/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
+++ b/llvm/test/CodeGen/AMDGPU/issue139317-bad-opsel-reg-sequence-fold.ll
@@ -18,21 +18,21 @@ define amdgpu_kernel void @stepper_test_kernel_DType_I6A6AcB6A6AsA6A6A_68a5362b9
; GFX942-NEXT: s_mov_b32 s9, 0x45004400
; GFX942-NEXT: s_mov_b32 s10, 0x42004000
; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-NEXT: v_mov_b64_e32 v[0:1], s[6:7]
; GFX942-NEXT: .LBB0_2: ; %.lr.ph
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_load_dwordx4 v[2:5], v0, s[2:3]
+; GFX942-NEXT: global_load_dwordx4 v[4:7], v2, s[2:3]
; GFX942-NEXT: s_add_u32 s4, s4, 8
; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[0:1]
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_pk_add_f16 v5, v5, s8
-; GFX942-NEXT: v_pk_add_f16 v4, v4, s9
-; GFX942-NEXT: v_pk_add_f16 v3, v3, s10
-; GFX942-NEXT: v_pk_add_f16 v2, v2, 1.0 op_sel:[0,1] op_sel_hi:[1,0]
-; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[0:1]
+; GFX942-NEXT: v_pk_add_f16 v7, v7, s8
+; GFX942-NEXT: v_pk_add_f16 v6, v6, s9
+; GFX942-NEXT: v_pk_add_f16 v5, v5, s10
+; GFX942-NEXT: v_pk_add_f16 v4, v4, 1.0 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX942-NEXT: global_store_dwordx4 v2, v[4:7], s[0:1]
; GFX942-NEXT: s_add_u32 s0, s0, 16
; GFX942-NEXT: s_addc_u32 s1, s1, 0
; GFX942-NEXT: s_add_u32 s2, s2, 16
diff --git a/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir b/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
index 6b5df1e9f5529..ec93b52458e64 100644
--- a/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
+++ b/llvm/test/CodeGen/AMDGPU/licm-regpressure.mir
@@ -32,46 +32,46 @@ body: |
; GCN-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GCN-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GCN-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GCN-NEXT: {{ $}}
- ; GCN-NEXT: bb.1:
- ; GCN-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
- ; GCN-NEXT: liveins: $vcc
- ; GCN-NEXT: {{ $}}
- ; GCN-NEXT: $vcc = S_AND_B64 $exec, $vcc, implicit-def $scc
; GCN-NEXT: [[V_CVT_F64_I32_e32_:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_1:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY1]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_1]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_2:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY2]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_2]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_3:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY3]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_3]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_4:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY4]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_4]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_5:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY5]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_5]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_6:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY6]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_6]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_7:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY7]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_7]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_8:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY8]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_8]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_9:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY9]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_9]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_10:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY10]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_10]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_11:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY11]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_11]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_12:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY12]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_12]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_13:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY13]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_13]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_14:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY14]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_14]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_15:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY15]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_15]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_16:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY16]], implicit $mode, implicit $exec
- ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_16]], implicit $exec
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: bb.1:
+ ; GCN-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+ ; GCN-NEXT: liveins: $vcc
+ ; GCN-NEXT: {{ $}}
+ ; GCN-NEXT: $vcc = S_AND_B64 $exec, $vcc, implicit-def $scc
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_1]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_2]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_3]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_4]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_5]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_6]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_7]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_8]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_9]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_10]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_11]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_12]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_13]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_14]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_15]], implicit $exec
+ ; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, [[V_CVT_F64_I32_e32_16]], implicit $exec
; GCN-NEXT: [[V_CVT_F64_I32_e32_17:%[0-9]+]]:vreg_64 = nofpexcept V_CVT_F64_I32_e32 [[COPY17]], implicit $mode, implicit $exec
; GCN-NEXT: $vcc = V_CMP_EQ_U64_e64 $vcc, killed [[V_CVT_F64_I32_e32_17]], implicit $exec
; GCN-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 79454e5c892da..4d68b98d6d590 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -339,13 +339,13 @@
; GCN-O1-NEXT: MachineDominator Tree Construction
; GCN-O1-NEXT: Machine Natural Loop Construction
; GCN-O1-NEXT: Machine Block Frequency Analysis
-; GCN-O1-NEXT: Machine Register Class Info Analysis
; GCN-O1-NEXT: Early Machine Loop Invariant Code Motion
; GCN-O1-NEXT: MachineDominator Tree Construction
; GCN-O1-NEXT: Machine Block Frequency Analysis
; GCN-O1-NEXT: Machine Common Subexpression Elimination
; GCN-O1-NEXT: MachinePostDominator Tree Construction
; GCN-O1-NEXT: Machine Cycle Info Analysis
+; GCN-O1-NEXT: Machine Register Class Info Analysis
; GCN-O1-NEXT: Machine code sinking
; GCN-O1-NEXT: Peephole Optimizations
; GCN-O1-NEXT: Remove dead machine instructions
@@ -659,13 +659,13 @@
; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction
; GCN-O1-OPTS-NEXT: Machine Natural Loop Construction
; GCN-O1-OPTS-NEXT: Machine Block Frequency Analysis
-; GCN-O1-OPTS-NEXT: Machine Register Class Info Analysis
; GCN-O1-OPTS-NEXT: Early Machine Loop Invariant Code Motion
; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction
; GCN-O1-OPTS-NEXT: Machine Block Frequency Analysis
; GCN-O1-OPTS-NEXT: Machine Common Subexpression Elimination
; GCN-O1-OPTS-NEXT: MachinePostDominator Tree Construction
; GCN-O1-OPTS-NEXT: Machine Cycle Info Analysis
+; GCN-O1-OPTS-NEXT: Machine Register Class Info Analysis
; GCN-O1-OPTS-NEXT: Machine code sinking
; GCN-O1-OPTS-NEXT: Peephole Optimizations
; GCN-O1-OPTS-NEXT: Remove dead machine instructions
@@ -990,13 +990,13 @@
; GCN-O2-NEXT: MachineDominator Tree Construction
; GCN-O2-NEXT: Machine Natural Loop Construction
; GCN-O2-NEXT: Machine Block Frequency Analysis
-; GCN-O2-NEXT: Machine Register Class Info Analysis
; GCN-O2-NEXT: Early Machine Loop Invariant Code Motion
; GCN-O2-NEXT: MachineDominator Tree Construction
; GCN-O2-NEXT: Machine Block Frequency Analysis
; GCN-O2-NEXT: Machine Common Subexpression Elimination
; GCN-O2-NEXT: MachinePostDominator Tree Construction
; GCN-O2-NEXT: Machine Cycle Info Analysis
+; GCN-O2-NEXT: Machine Register Class Info Analysis
; GCN-O2-NEXT: Machine code sinking
; GCN-O2-NEXT: Peephole Optimizations
; GCN-O2-NEXT: Remove dead machine instructions
@@ -1337,13 +1337,13 @@
; GCN-O3-NEXT: MachineDominator Tree Construction
; GCN-O3-NEXT: Machine Natural Loop Construction
; GCN-O3-NEXT: Machine Block Frequency Analysis
-; GCN-O3-NEXT: Machine Register Class Info Analysis
; GCN-O3-NEXT: Early Machine Loop Invariant Code Motion
; GCN-O3-NEXT: MachineDominator Tree Construction
; GCN-O3-NEXT: Machine Block Frequency Analysis
; GCN-O3-NEXT: Machine Common Subexpression Elimination
; GCN-O3-NEXT: MachinePostDominator Tree Construction
; GCN-O3-NEXT: Machine Cycle Info Analysis
+; GCN-O3-NEXT: Machine Register Class Info Analysis
; GCN-O3-NEXT: Machine code sinking
; GCN-O3-NEXT: Peephole Optimizations
; GCN-O3-NEXT: Remove dead machine instructions
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
index 530d8b8b4a1fb..432c728e4e8a8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
@@ -14,20 +14,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32(<4 x i32> %addr, i32 %i
; GFX11-LABEL: struct_atomic_buffer_load_i32:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB0_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -38,18 +38,18 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32(<4 x i32> %addr, i32 %i
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB0_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -123,20 +123,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i3
; GFX11-LABEL: struct_atomic_buffer_load_i32_off:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB2_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -147,18 +147,18 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_off(<4 x i32> %addr, i3
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB2_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -180,20 +180,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i
; GFX11-LABEL: struct_atomic_buffer_load_i32_soff:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB3_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB3_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -204,19 +204,19 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_soff(<4 x i32> %addr, i
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT: .LBB3_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -237,20 +237,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i3
; GFX11-LABEL: struct_atomic_buffer_load_i32_dlc:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB4_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB4_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -261,18 +261,18 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i32_dlc(<4 x i32> %addr, i3
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT: .LBB4_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -352,20 +352,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX11-LABEL: struct_atomic_buffer_load_i64:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: .LBB6_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -376,19 +376,19 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
-; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -402,19 +402,19 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
-; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -428,19 +428,19 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -454,19 +454,19 @@ define amdgpu_kernel void @struct_atomic_buffer_load_i64(<4 x i32> %addr, i32 %i
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -489,20 +489,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32
; GFX11-LABEL: struct_atomic_buffer_load_v2i16:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB7_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -513,18 +513,18 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v2i16(<4 x i32> %addr, i32
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB7_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB7_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -547,21 +547,21 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-SDAG-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
@@ -569,23 +569,23 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-SDAG-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
@@ -593,23 +593,23 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-GISEL-TRUE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
@@ -617,23 +617,23 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-GISEL-FAKE16-LABEL: struct_atomic_buffer_load_v4i16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
@@ -641,23 +641,23 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX11-GISEL-LABEL: struct_atomic_buffer_load_v4i16:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
@@ -668,15 +668,15 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
-; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
@@ -694,21 +694,21 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
-; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -722,20 +722,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s7
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -750,20 +750,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i16(<4 x i32> %addr, i32
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s7
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -788,20 +788,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32
; GFX11-LABEL: struct_atomic_buffer_load_v4i32:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB9_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b128 v[1:4], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -812,15 +812,15 @@ define amdgpu_kernel void @struct_atomic_buffer_load_v4i32(<4 x i32> %addr, i32
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-NEXT: .LBB9_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
@@ -846,22 +846,22 @@ define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %i
; GFX11-LABEL: struct_atomic_buffer_load_ptr:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB10_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-NEXT: flat_load_b32 v2, v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -872,20 +872,20 @@ define amdgpu_kernel void @struct_atomic_buffer_load_ptr(<4 x i32> %addr, i32 %i
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB10_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-NEXT: .LBB10_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-NEXT: flat_load_b32 v2, v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
index 099fb7b893a95..151c7b1795cbd 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
@@ -14,20 +14,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32(ptr addrspace(8) %p
; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB0_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -38,18 +38,18 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32(ptr addrspace(8) %p
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB0_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB0_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -123,20 +123,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8
; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_off:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB2_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB2_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -147,18 +147,18 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_off(ptr addrspace(8
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB2_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB2_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -180,20 +180,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(
; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_soff:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB3_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 4 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 4 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB3_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -204,19 +204,19 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_soff(ptr addrspace(
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_mov_b32 s5, 4
-; GFX12-NEXT: .LBB3_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
+; GFX12-NEXT: .LBB3_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], s5 idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -237,20 +237,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8
; GFX11-LABEL: struct_ptr_atomic_buffer_load_i32_dlc:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB4_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen offset:4 dlc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen offset:4 dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB4_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -261,18 +261,18 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i32_dlc(ptr addrspace(8
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB4_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
+; GFX12-NEXT: .LBB4_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT_RT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -352,20 +352,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX11-LABEL: struct_ptr_atomic_buffer_load_i64:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: .LBB6_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b64 v[3:4], v2, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[3:4], v[0:1]
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB6_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -376,19 +376,19 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
-; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -402,19 +402,19 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
-; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-SDAG-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -428,19 +428,19 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-TRUE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -454,19 +454,19 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_i64(ptr addrspace(8) %p
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
-; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s6
-; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
+; GFX12-GISEL-FAKE16-NEXT: .LBB6_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[4:5], v2, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[2:3], v[0:1]
+; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u64_e32 vcc_lo, v[4:5], v[0:1]
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -489,20 +489,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8)
; GFX11-LABEL: struct_ptr_atomic_buffer_load_v2i16:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB7_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b32 v1, v1, s[0:3], 0 idxen glc
+; GFX11-NEXT: buffer_load_b32 v2, v1, s[0:3], 0 idxen glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB7_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -513,18 +513,18 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v2i16(ptr addrspace(8)
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB7_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
-; GFX12-NEXT: buffer_load_b32 v1, v1, s[0:3], null idxen th:TH_LOAD_NT
+; GFX12-NEXT: .LBB7_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: buffer_load_b32 v2, v1, s[0:3], null idxen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -547,21 +547,21 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-SDAG-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_clause 0x1
+; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX11-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-SDAG-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-SDAG-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-SDAG-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-TRUE16-NEXT: s_endpgm
@@ -569,23 +569,23 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-SDAG-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %bb
; GFX11-SDAG-FAKE16-NEXT: s_clause 0x1
+; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-SDAG-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX11-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-SDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX11-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-SDAG-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-SDAG-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-SDAG-FAKE16-NEXT: s_endpgm
@@ -593,23 +593,23 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-GISEL-TRUE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX11-GISEL-TRUE16: ; %bb.0: ; %bb
; GFX11-GISEL-TRUE16-NEXT: s_clause 0x1
+; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-TRUE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-TRUE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-TRUE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-TRUE16-NEXT: s_endpgm
@@ -617,23 +617,23 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-GISEL-FAKE16-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX11-GISEL-FAKE16: ; %bb.0: ; %bb
; GFX11-GISEL-FAKE16-NEXT: s_clause 0x1
+; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-FAKE16-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-FAKE16-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-FAKE16-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-FAKE16-NEXT: s_endpgm
@@ -641,23 +641,23 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX11-GISEL-LABEL: struct_ptr_atomic_buffer_load_v4i16:
; GFX11-GISEL: ; %bb.0: ; %bb
; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: s_mov_b32 s5, 0
+; GFX11-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX11-GISEL-NEXT: .LBB8_1: ; %bb1
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-GISEL-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-GISEL-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v1
-; GFX11-GISEL-NEXT: v_readfirstlane_b32 s7, v2
-; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s5, v2
+; GFX11-GISEL-NEXT: v_readfirstlane_b32 s6, v3
+; GFX11-GISEL-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s6, v0
-; GFX11-GISEL-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-GISEL-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
+; GFX11-GISEL-NEXT: s_or_b32 s4, vcc_lo, s4
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB8_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %bb2
; GFX11-GISEL-NEXT: s_endpgm
@@ -668,15 +668,15 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-SDAG-TRUE16-NEXT: v_nop
; GFX12-SDAG-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-TRUE16-NEXT: s_clause 0x1
-; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
@@ -694,21 +694,21 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-SDAG-FAKE16-NEXT: v_nop
; GFX12-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-SDAG-FAKE16-NEXT: s_clause 0x1
-; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-SDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-SDAG-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-SDAG-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-SDAG-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-SDAG-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v1, v3, 16, v1
-; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-SDAG-FAKE16-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX12-SDAG-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-SDAG-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-SDAG-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
@@ -722,20 +722,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-GISEL-TRUE16-NEXT: v_nop
; GFX12-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-TRUE16-NEXT: s_clause 0x1
-; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-TRUE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-TRUE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-TRUE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s7
+; GFX12-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GISEL-TRUE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
; GFX12-GISEL-TRUE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -750,20 +750,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i16(ptr addrspace(8)
; GFX12-GISEL-FAKE16-NEXT: v_nop
; GFX12-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-GISEL-FAKE16-NEXT: s_clause 0x1
-; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-GISEL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
-; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-GISEL-FAKE16-NEXT: .LBB8_1: ; %bb1
+; GFX12-GISEL-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-GISEL-FAKE16-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s7, v3
-; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s7
+; GFX12-GISEL-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX12-GISEL-FAKE16-NEXT: s_pack_ll_b32_b16 s5, s5, s6
; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-GISEL-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, s5, v0
; GFX12-GISEL-FAKE16-NEXT: s_or_b32 s4, vcc_lo, s4
@@ -788,20 +788,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8)
; GFX11-LABEL: struct_ptr_atomic_buffer_load_v4i32:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB9_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b128 v[1:4], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v4, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB9_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -812,15 +812,15 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_v4i32(ptr addrspace(8)
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB9_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-NEXT: .LBB9_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: buffer_load_b128 v[2:5], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v5, v0
@@ -846,22 +846,22 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %p
; GFX11-LABEL: struct_ptr_atomic_buffer_load_ptr:
; GFX11: ; %bb.0: ; %bb
; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: .LBB10_1: ; %bb1
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-NEXT: buffer_load_b64 v[1:2], v1, s[0:3], 0 idxen offset:4 glc
+; GFX11-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], 0 idxen offset:4 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: flat_load_b32 v1, v[1:2]
+; GFX11-NEXT: flat_load_b32 v2, v[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
+; GFX11-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
; GFX11-NEXT: s_cbranch_execnz .LBB10_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -872,20 +872,20 @@ define amdgpu_kernel void @struct_ptr_atomic_buffer_load_ptr(ptr addrspace(8) %p
; GFX12-NEXT: v_nop
; GFX12-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX12-NEXT: s_wait_xcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, 0
-; GFX12-NEXT: .LBB10_1: ; %bb1
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s6
+; GFX12-NEXT: .LBB10_1: ; %bb1
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: buffer_load_b64 v[2:3], v1, s[0:3], null idxen offset:4 th:TH_LOAD_NT
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: flat_load_b32 v1, v[2:3]
+; GFX12-NEXT: flat_load_b32 v2, v[2:3]
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v1, v0
+; GFX12-NEXT: v_cmp_ne_u32_e32 vcc_lo, v2, v0
; GFX12-NEXT: s_or_b32 s4, vcc_lo, s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s4
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index 71e5da3447c96..6f04bb7f10e33 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -6290,42 +6290,43 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6340,38 +6341,39 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6386,33 +6388,33 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_add_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_1
@@ -6426,38 +6428,38 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fadd_ret_v2bf16:
@@ -6465,29 +6467,29 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6505,30 +6507,30 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6654,42 +6656,43 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6704,38 +6707,39 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6750,33 +6754,33 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_add_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_add_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB25_1
@@ -6790,38 +6794,38 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_add_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fadd_ret_v2bf16__offset:
@@ -6829,29 +6833,29 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_add_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6869,30 +6873,30 @@ define <2 x bfloat> @local_atomic_fadd_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_add_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_add_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7020,42 +7024,42 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7068,39 +7072,39 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7113,34 +7117,34 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX10-LABEL: local_atomic_fadd_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: ds_read_b32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB26_1
@@ -7151,35 +7155,35 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX90A-LABEL: local_atomic_fadd_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7189,35 +7193,35 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX908-LABEL: local_atomic_fadd_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0
+; GFX908-NEXT: ds_read_b32 v3, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7228,36 +7232,36 @@ define void @local_atomic_fadd_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7369,42 +7373,42 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7417,39 +7421,39 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-FAKE16-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_add_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7462,34 +7466,34 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX10-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_add_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB27_1
@@ -7500,35 +7504,35 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX90A-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7538,35 +7542,35 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX908-LABEL: local_atomic_fadd_noret_v2bf16__ofset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX908-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7577,36 +7581,36 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_add_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_add_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_add_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_add_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8316,22 +8320,21 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB28_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s10, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: s_lshl_b32 s8, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
+; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_2
@@ -8347,23 +8350,22 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_7
; GFX7-NEXT: ; %bb.5:
-; GFX7-NEXT: s_lshl_b32 s3, s3, 4
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: ds_read_b32 v2, v1
+; GFX7-NEXT: s_lshl_b32 s0, s3, 4
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
; GFX7-NEXT: .LBB28_7: ; %Flow21
@@ -8398,24 +8400,23 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_13
; GFX7-NEXT: ; %bb.10:
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mov_b32_e32 v3, s2
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v2
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: ds_read_b32 v2, v3
+; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB28_11: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v1
-; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v2, v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_add_f32_e32 v2, v4, v1
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v3
-; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
+; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB28_11
; GFX7-NEXT: ; %bb.12: ; %Flow
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: .LBB28_13: ; %Flow19
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
@@ -8443,22 +8444,21 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB28_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s10, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v1, s10
-; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: s_lshl_b32 s8, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: ds_read_b32 v1, v2
; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
+; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: v_mov_b32_e32 v1, s10
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v1
+; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_2
@@ -8474,23 +8474,22 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB28_7
; GFX6-NEXT: ; %bb.5:
-; GFX6-NEXT: s_lshl_b32 s3, s3, 4
-; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: ds_read_b32 v2, v1
+; GFX6-NEXT: s_lshl_b32 s0, s3, 4
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_read_b32 v3, v1
; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB28_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX6-NEXT: v_mov_b32_e32 v4, s3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB28_6
; GFX6-NEXT: .LBB28_7: ; %Flow19
@@ -8525,24 +8524,23 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX6-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB28_13
; GFX6-NEXT: ; %bb.10:
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: ds_read_b32 v2, v3
+; GFX6-NEXT: s_mov_b64 s[2:3], 0
; GFX6-NEXT: .LBB28_11: ; %atomicrmw.start8
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v3, v2
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v1
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v2, v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v2
+; GFX6-NEXT: v_add_f32_e32 v2, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v3
-; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
+; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX6-NEXT: s_cbranch_execnz .LBB28_11
; GFX6-NEXT: ; %bb.12: ; %Flow
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX6-NEXT: .LBB28_13: ; %Flow17
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
@@ -9150,22 +9148,21 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX7-NEXT: s_cbranch_execz .LBB29_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s10, s3, 3
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: ds_read_b32 v1, v1
+; GFX7-NEXT: s_lshl_b32 s8, s3, 3
+; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
+; GFX7-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v1
+; GFX7-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX7-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_2
@@ -9181,23 +9178,22 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_7
; GFX7-NEXT: ; %bb.5:
-; GFX7-NEXT: s_lshl_b32 s3, s3, 4
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: ds_read_b32 v2, v1
+; GFX7-NEXT: s_lshl_b32 s0, s3, 4
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX7-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GFX7-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX7-NEXT: s_mov_b64 s[8:9], 0
; GFX7-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
-; GFX7-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX7-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX7-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: v_mov_b32_e32 v2, v3
+; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_6
; GFX7-NEXT: .LBB29_7: ; %Flow21
@@ -9232,24 +9228,23 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_13
; GFX7-NEXT: ; %bb.10:
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mov_b32_e32 v3, s2
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_b32 v2, v2
-; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: ds_read_b32 v2, v3
+; GFX7-NEXT: s_mov_b64 s[2:3], 0
; GFX7-NEXT: .LBB29_11: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_add_f32_e32 v4, v3, v1
-; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v2, v3, v4
+; GFX7-NEXT: v_mov_b32_e32 v4, v2
+; GFX7-NEXT: v_add_f32_e32 v2, v4, v1
+; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v3
-; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
+; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_cbranch_execnz .LBB29_11
; GFX7-NEXT: ; %bb.12: ; %Flow
-; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: .LBB29_13: ; %Flow19
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
@@ -9277,22 +9272,21 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX6-NEXT: s_cbranch_execz .LBB29_4
; GFX6-NEXT: ; %bb.1:
-; GFX6-NEXT: s_lshl_b32 s10, s3, 3
-; GFX6-NEXT: v_mov_b32_e32 v1, s10
-; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: s_lshl_b32 s8, s3, 3
+; GFX6-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NEXT: ds_read_b32 v1, v2
; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
-; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v3, s0
+; GFX6-NEXT: v_mul_f32_e32 v3, 0x42280000, v3
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_2: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v3, v1
-; GFX6-NEXT: v_mov_b32_e32 v1, s10
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
-; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v1, v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v1
+; GFX6-NEXT: v_add_f32_e32 v1, v4, v3
+; GFX6-NEXT: ds_cmpst_rtn_b32 v1, v2, v4, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v3
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v4
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_2
@@ -9308,23 +9302,22 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB29_7
; GFX6-NEXT: ; %bb.5:
-; GFX6-NEXT: s_lshl_b32 s3, s3, 4
-; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: ds_read_b32 v2, v1
+; GFX6-NEXT: s_lshl_b32 s0, s3, 4
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_read_b32 v3, v1
; GFX6-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX6-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GFX6-NEXT: v_mul_f32_e32 v2, 0x42280000, v2
; GFX6-NEXT: s_mov_b64 s[8:9], 0
; GFX6-NEXT: .LBB29_6: ; %atomicrmw.start2
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_add_f32_e32 v3, v2, v1
-; GFX6-NEXT: v_mov_b32_e32 v4, s3
-; GFX6-NEXT: ds_cmpst_rtn_b32 v3, v4, v2, v3
+; GFX6-NEXT: v_add_f32_e32 v4, v3, v2
+; GFX6-NEXT: ds_cmpst_rtn_b32 v4, v1, v3, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v3, v2
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v4, v3
; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v3, v4
; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX6-NEXT: s_cbranch_execnz .LBB29_6
; GFX6-NEXT: .LBB29_7: ; %Flow19
@@ -9359,24 +9352,23 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX6-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
; GFX6-NEXT: s_cbranch_execz .LBB29_13
; GFX6-NEXT: ; %bb.10:
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_mov_b32_e32 v3, s2
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: s_mov_b64 s[8:9], 0
+; GFX6-NEXT: ds_read_b32 v2, v3
+; GFX6-NEXT: s_mov_b64 s[2:3], 0
; GFX6-NEXT: .LBB29_11: ; %atomicrmw.start8
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v3, v2
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: v_add_f32_e32 v4, v3, v1
-; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v2, v3, v4
+; GFX6-NEXT: v_mov_b32_e32 v4, v2
+; GFX6-NEXT: v_add_f32_e32 v2, v4, v1
+; GFX6-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v3
-; GFX6-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
+; GFX6-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX6-NEXT: s_cbranch_execnz .LBB29_11
; GFX6-NEXT: ; %bb.12: ; %Flow
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX6-NEXT: .LBB29_13: ; %Flow17
; GFX6-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
index 8dbddc160a8d2..81d26723b336b 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmax.ll
@@ -5035,15 +5035,15 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v4, v3, v3
-; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5064,38 +5064,38 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v3, v3
+; GFX942-NEXT: v_pk_max_f16 v1, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v2, v4, v2
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v2
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB20_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: local_atomic_fmax_ret_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX11-NEXT: v_pk_max_f16 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5114,14 +5114,14 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX10-NEXT: v_pk_max_f16 v2, v4, v2
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5140,22 +5140,22 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v2
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_pk_max_f16 v1, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v2
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmax_ret_v2f16:
@@ -5163,16 +5163,16 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v2, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v2, v2, v3
-; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v1
+; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB20_1
@@ -5187,17 +5187,17 @@ define <2 x half> @local_atomic_fmax_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v2, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -5298,15 +5298,15 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v4, v3, v3
-; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5327,38 +5327,38 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v3, v3
+; GFX942-NEXT: v_pk_max_f16 v1, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v2, v4, v2
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v2
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: local_atomic_fmax_ret_v2f16__offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX11-NEXT: v_pk_max_f16 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX11-NEXT: v_pk_max_f16 v2, v2, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5377,14 +5377,14 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX10-NEXT: v_pk_max_f16 v2, v4, v2
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_max_f16 v2, v2, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5403,22 +5403,22 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX90A-NEXT: v_pk_max_f16 v2, v4, v2
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_pk_max_f16 v1, v3, v3
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v2
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmax_ret_v2f16__offset:
@@ -5426,16 +5426,16 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v2, v4, v4
-; GFX908-NEXT: v_pk_max_f16 v2, v2, v3
-; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_max_f16 v2, v2, v1
+; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
@@ -5450,17 +5450,17 @@ define <2 x half> @local_atomic_fmax_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v2, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_max_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX8-NEXT: v_max_f16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -5563,14 +5563,14 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v1, v1
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v3
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5591,13 +5591,13 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v2, v0
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v1
; GFX942-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v3, v3, v1
; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5613,14 +5613,14 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v1, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5639,13 +5639,13 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v1, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5664,12 +5664,12 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v2, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v1
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX90A-NEXT: v_pk_max_f16 v3, v3, v1
; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5686,12 +5686,12 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v1
; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5707,23 +5707,23 @@ define void @local_atomic_fmax_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v4, v6, v5
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v4, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v4, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5812,14 +5812,14 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v1, v1
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v3
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5840,13 +5840,13 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v1
; GFX942-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX942-NEXT: v_pk_max_f16 v3, v3, v1
; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5862,14 +5862,14 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v1, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX11-NEXT: v_pk_max_f16 v3, v3, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5888,13 +5888,13 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v1, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX10-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX10-NEXT: v_pk_max_f16 v3, v3, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5913,12 +5913,12 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v1
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX90A-NEXT: v_pk_max_f16 v3, v3, v1
; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5935,12 +5935,12 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v4, v3
+; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX908-NEXT: v_pk_max_f16 v3, v3, v1
; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5956,23 +5956,23 @@ define void @local_atomic_fmax_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v4, v6, v5
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_max_f16_sdwa v4, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v3, v3
+; GFX8-NEXT: v_max_f16_sdwa v4, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_max_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6068,41 +6068,43 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6121,38 +6123,39 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6168,81 +6171,82 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fmax_ret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6257,38 +6261,39 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6303,33 +6308,33 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_max_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_1
@@ -6343,38 +6348,38 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmax_ret_v2bf16:
@@ -6382,29 +6387,29 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6422,30 +6427,30 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6555,41 +6560,43 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6608,38 +6615,39 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6655,81 +6663,82 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fmax_ret_v2bf16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6744,38 +6753,39 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6790,33 +6800,33 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_max_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_max_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB25_1
@@ -6830,38 +6840,38 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_max_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmax_ret_v2bf16__offset:
@@ -6869,29 +6879,29 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_max_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6909,30 +6919,30 @@ define <2 x bfloat> @local_atomic_fmax_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_max_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_max_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7044,42 +7054,41 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7096,39 +7105,39 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7141,36 +7150,36 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX942-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB26_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7180,42 +7189,42 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_max_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_max_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7228,39 +7237,39 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-FAKE16-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7273,34 +7282,34 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX10-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: ds_read_b32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB26_1
@@ -7311,35 +7320,35 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX90A-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7349,35 +7358,35 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX908-LABEL: local_atomic_fmax_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0
+; GFX908-NEXT: ds_read_b32 v3, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7388,36 +7397,36 @@ define void @local_atomic_fmax_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7513,42 +7522,41 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_max_num_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7565,39 +7573,39 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_max_num_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7610,36 +7618,36 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX942-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7649,42 +7657,42 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_max_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_max_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7697,39 +7705,39 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-FAKE16-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_max_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_max_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7742,34 +7750,34 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX10-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_max_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB27_1
@@ -7780,35 +7788,35 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX90A-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7818,35 +7826,35 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX908-LABEL: local_atomic_fmax_noret_v2bf16__ofset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX908-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7857,36 +7865,36 @@ define void @local_atomic_fmax_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_max_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_max_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_max_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_max_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
index 8e2c1dad78297..3b2cff7027681 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fmin.ll
@@ -5035,15 +5035,15 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v4, v3, v3
-; GFX12-NEXT: v_pk_min_num_f16 v2, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5064,38 +5064,38 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v3, v3
+; GFX942-NEXT: v_pk_max_f16 v1, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v2, v4, v2
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX942-NEXT: v_pk_min_f16 v1, v1, v2
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB20_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: local_atomic_fmin_ret_v2f16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX11-NEXT: v_pk_min_f16 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5114,14 +5114,14 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX10-NEXT: v_pk_min_f16 v2, v4, v2
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5140,22 +5140,22 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v4, v2
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_pk_max_f16 v1, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v1, v1, v2
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB20_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmin_ret_v2f16:
@@ -5163,16 +5163,16 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v2, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v2, v2, v3
-; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v1
+; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB20_1
@@ -5187,17 +5187,17 @@ define <2 x half> @local_atomic_fmin_ret_v2f16(ptr addrspace(3) %ptr, <2 x half>
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v2, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -5298,15 +5298,15 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-NEXT: v_pk_max_num_f16 v2, v1, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v4, v3, v3
-; GFX12-NEXT: v_pk_min_num_f16 v2, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5327,38 +5327,38 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v2, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v3, v3
+; GFX942-NEXT: v_pk_max_f16 v1, v3, v3
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v2, v4, v2
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX942-NEXT: v_pk_min_f16 v1, v1, v2
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_cbranch_execnz .LBB21_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: local_atomic_fmin_ret_v2f16__offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX11-NEXT: v_pk_min_f16 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX11-NEXT: v_pk_min_f16 v2, v2, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5377,14 +5377,14 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_pk_max_f16 v2, v1, v1
-; GFX10-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX10-NEXT: v_pk_min_f16 v2, v4, v2
+; GFX10-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX10-NEXT: v_pk_min_f16 v2, v2, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5403,22 +5403,22 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v2, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v3, v3
-; GFX90A-NEXT: v_pk_min_f16 v2, v4, v2
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_pk_max_f16 v1, v3, v3
+; GFX90A-NEXT: v_pk_min_f16 v1, v1, v2
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX90A-NEXT: s_cbranch_execnz .LBB21_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmin_ret_v2f16__offset:
@@ -5426,16 +5426,16 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
-; GFX908-NEXT: v_pk_max_f16 v2, v4, v4
-; GFX908-NEXT: v_pk_min_f16 v2, v2, v3
-; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
+; GFX908-NEXT: v_mov_b32_e32 v3, v2
+; GFX908-NEXT: v_pk_max_f16 v2, v3, v3
+; GFX908-NEXT: v_pk_min_f16 v2, v2, v1
+; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
; GFX908-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX908-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX908-NEXT: s_cbranch_execnz .LBB21_1
@@ -5450,17 +5450,17 @@ define <2 x half> @local_atomic_fmin_ret_v2f16__offset(ptr addrspace(3) %ptr, <2
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v2, v1, v1
-; GFX8-NEXT: v_max_f16_sdwa v5, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v6, v4, v4
-; GFX8-NEXT: v_min_f16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v4, v4
+; GFX8-NEXT: v_min_f16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -5563,14 +5563,14 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v1, v1
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v3
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5591,13 +5591,13 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v2, v0
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v1
; GFX942-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v3, v3, v1
; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5613,14 +5613,14 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v1, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5639,13 +5639,13 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v1, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX10-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5664,12 +5664,12 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v2, v0
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v1
; GFX90A-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX90A-NEXT: v_pk_min_f16 v3, v3, v1
; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5686,12 +5686,12 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v1
; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5707,23 +5707,23 @@ define void @local_atomic_fmin_noret_v2f16(ptr addrspace(3) %ptr, <2 x half> %va
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_min_f16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v4, v6, v5
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_max_f16_sdwa v4, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v4, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB22_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -5812,14 +5812,14 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_pk_max_num_f16 v3, v1, v1
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v3, v2, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v3
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v1
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -5840,13 +5840,13 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_pk_max_f16 v1, v1, v1
; GFX942-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_pk_max_f16 v3, v1, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX942-NEXT: v_pk_max_f16 v3, v2, v2
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX942-NEXT: v_pk_min_f16 v3, v3, v1
; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5862,14 +5862,14 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-NEXT: v_pk_max_f16 v1, v1, v1
; GFX11-NEXT: s_mov_b32 s0, 0
; GFX11-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: v_pk_max_f16 v3, v1, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_max_f16 v4, v2, v2
+; GFX11-NEXT: v_pk_max_f16 v3, v2, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX11-NEXT: v_pk_min_f16 v3, v3, v1
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5888,13 +5888,13 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_pk_max_f16 v1, v1, v1
; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_pk_max_f16 v3, v1, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX10-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX10-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX10-NEXT: v_pk_min_f16 v3, v3, v1
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -5913,12 +5913,12 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[4:5], 0
+; GFX90A-NEXT: v_pk_max_f16 v1, v1, v1
; GFX90A-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_pk_max_f16 v3, v1, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX90A-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX90A-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX90A-NEXT: v_pk_min_f16 v3, v3, v1
; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5935,12 +5935,12 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[4:5], 0
+; GFX908-NEXT: v_pk_max_f16 v1, v1, v1
; GFX908-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_pk_max_f16 v3, v1, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_pk_max_f16 v4, v2, v2
-; GFX908-NEXT: v_pk_min_f16 v3, v4, v3
+; GFX908-NEXT: v_pk_max_f16 v3, v2, v2
+; GFX908-NEXT: v_pk_min_f16 v3, v3, v1
; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
@@ -5956,23 +5956,23 @@ define void @local_atomic_fmin_noret_v2f16__offset(ptr addrspace(3) %ptr, <2 x h
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[4:5], 0
+; GFX8-NEXT: v_max_f16_sdwa v2, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v1, v1, v1
; GFX8-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_max_f16_sdwa v3, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_max_f16_sdwa v4, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v5, v1, v1
-; GFX8-NEXT: v_max_f16_e32 v6, v2, v2
-; GFX8-NEXT: v_min_f16_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_f16_e32 v4, v6, v5
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_max_f16_sdwa v4, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_max_f16_e32 v5, v3, v3
+; GFX8-NEXT: v_min_f16_sdwa v4, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_min_f16_e32 v5, v5, v1
+; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_cbranch_execnz .LBB23_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -6068,41 +6068,43 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6121,38 +6123,39 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6168,81 +6171,82 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fmin_ret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6257,38 +6261,39 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6303,33 +6308,33 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_min_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_1
@@ -6343,38 +6348,38 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmin_ret_v2bf16:
@@ -6382,29 +6387,29 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6422,30 +6427,30 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6555,41 +6560,43 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6608,38 +6615,39 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6655,81 +6663,82 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fmin_ret_v2bf16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6744,38 +6753,39 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -6790,33 +6800,33 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_min_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_min_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB25_1
@@ -6830,38 +6840,38 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_min_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fmin_ret_v2bf16__offset:
@@ -6869,29 +6879,29 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_min_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -6909,30 +6919,30 @@ define <2 x bfloat> @local_atomic_fmin_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_min_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_min_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7044,42 +7054,41 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7096,39 +7105,39 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7141,36 +7150,36 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX942-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB26_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7180,42 +7189,42 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_min_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_min_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7228,39 +7237,39 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-FAKE16-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7273,34 +7282,34 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX10-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: ds_read_b32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB26_1
@@ -7311,35 +7320,35 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX90A-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7349,35 +7358,35 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX908-LABEL: local_atomic_fmin_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0
+; GFX908-NEXT: ds_read_b32 v3, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7388,36 +7397,36 @@ define void @local_atomic_fmin_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7513,42 +7522,41 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_min_num_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7565,39 +7573,39 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_min_num_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7610,36 +7618,36 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX942-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7649,42 +7657,42 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_min_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_min_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7697,39 +7705,39 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-FAKE16-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_min_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_min_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7742,34 +7750,34 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX10-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_min_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB27_1
@@ -7780,35 +7788,35 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX90A-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7818,35 +7826,35 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX908-LABEL: local_atomic_fmin_noret_v2bf16__ofset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX908-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7857,36 +7865,36 @@ define void @local_atomic_fmin_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_min_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_min_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_min_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_min_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
index 4536ded237941..1b1d2299af839 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fsub.ll
@@ -6854,41 +6854,43 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6907,38 +6909,39 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6954,81 +6957,82 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB24_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fsub_ret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7043,38 +7047,39 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7089,33 +7094,33 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_sub_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB24_1
@@ -7129,38 +7134,38 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB24_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fsub_ret_v2bf16:
@@ -7168,29 +7173,29 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7208,30 +7213,30 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16(ptr addrspace(3) %ptr, <2 x bf
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7341,41 +7346,43 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7394,38 +7401,39 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
+; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX12-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7441,81 +7449,82 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX942-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX942-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v2, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v2, v2
+; GFX942-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX942-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v1, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v1, v1
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v2, v4, v2, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v1, v5, v1, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: v_mov_b32_e32 v3, v1
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB25_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v0, v2
+; GFX942-NEXT: v_mov_b32_e32 v0, v1
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: local_atomic_fsub_ret_v2bf16__offset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_and_b32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v4, v3 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v5, v4 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7530,38 +7539,39 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_lshlrev_b32 v2, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v4, v6, v4 :: v_dual_lshlrev_b32 v5, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v5, v7, v9 :: v_dual_lshlrev_b32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v5, v7, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v6, v8, s0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v3 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v2, v0, v2, v4 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -7576,33 +7586,33 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v2
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX10-NEXT: v_sub_f32_e32 v2, v5, v2
-; GFX10-NEXT: v_sub_f32_e32 v4, v6, v4
-; GFX10-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX10-NEXT: v_mov_b32_e32 v4, v2
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX10-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX10-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX10-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX10-NEXT: v_cmp_u_f32_e64 s4, v2, v2
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v7, s4
-; GFX10-NEXT: v_perm_b32 v2, v4, v2, 0x7060302
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB25_1
@@ -7616,38 +7626,38 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
-; GFX90A-NEXT: v_sub_f32_e32 v2, v4, v2
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v2, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
-; GFX90A-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v2, v4, v2, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v2, v0, v3, v2 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX90A-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v1, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v1, v1
+; GFX90A-NEXT: v_cndmask_b32_e64 v1, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v1, v5, v1, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v1, v0, v3, v1 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v3, v2
+; GFX90A-NEXT: v_mov_b32_e32 v3, v1
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB25_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v0, v2
+; GFX90A-NEXT: v_mov_b32_e32 v0, v1
; GFX90A-NEXT: s_setpc_b64 s[30:31]
;
; GFX908-LABEL: local_atomic_fsub_ret_v2bf16__offset:
@@ -7655,29 +7665,29 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v4, v2
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX908-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX908-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX908-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX908-NEXT: v_sub_f32_e32 v2, v6, v2
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v2, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v2, v3, v2, s9
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX908-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v2, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX908-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v2, v5, v2, s9
; GFX908-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7695,30 +7705,30 @@ define <2 x bfloat> @local_atomic_fsub_ret_v2bf16__offset(ptr addrspace(3) %ptr,
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v4, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
-; GFX8-NEXT: v_sub_f32_e32 v3, v5, v3
-; GFX8-NEXT: v_sub_f32_e32 v2, v6, v2
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v2
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_alignbit_b32 v2, v3, v2, 16
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v4
+; GFX8-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v2
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v2, v2
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v2, v5, v2, 16
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v0, v4, v2 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
@@ -7830,42 +7840,41 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7882,39 +7891,39 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7927,36 +7936,36 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX942-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0
+; GFX942-NEXT: ds_read_b32 v3, v0
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB26_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -7966,42 +7975,42 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-TRUE16-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8014,39 +8023,39 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX11-FAKE16-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -8059,34 +8068,34 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX10-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0
+; GFX10-NEXT: ds_read_b32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB26_1
@@ -8097,35 +8106,35 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX90A-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0
+; GFX90A-NEXT: ds_read_b32 v3, v0
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB26_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8135,35 +8144,35 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX908-LABEL: local_atomic_fsub_noret_v2bf16:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0
+; GFX908-NEXT: ds_read_b32 v3, v0
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB26_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8174,36 +8183,36 @@ define void @local_atomic_fsub_noret_v2bf16(ptr addrspace(3) %ptr, <2 x bfloat>
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0
+; GFX8-NEXT: ds_read_b32 v3, v0
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB26_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB26_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8299,42 +8308,41 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX12-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX12-TRUE16-NEXT: s_wait_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8351,39 +8359,39 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX12-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX12-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX12-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX12-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX12-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX12-FAKE16-NEXT: s_wait_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SE
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8396,36 +8404,36 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX942-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX942-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX942-NEXT: s_mov_b64 s[2:3], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX942-NEXT: s_movk_i32 s4, 0x7fff
+; GFX942-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX942-NEXT: s_mov_b32 s5, 0x7060302
; GFX942-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX942-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX942-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX942-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX942-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX942-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX942-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX942-NEXT: v_add3_u32 v5, v5, v3, s4
-; GFX942-NEXT: v_add3_u32 v7, v7, v4, s4
-; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v3, v3
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX942-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX942-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX942-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX942-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX942-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX942-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX942-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX942-NEXT: v_add3_u32 v6, v6, v4, s4
+; GFX942-NEXT: v_add3_u32 v8, v8, v5, s4
+; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX942-NEXT: v_cmp_u_f32_e64 s[0:1], v4, v4
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX942-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[0:1]
-; GFX942-NEXT: v_perm_b32 v3, v4, v3, s5
-; GFX942-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX942-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX942-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[0:1]
+; GFX942-NEXT: v_perm_b32 v4, v5, v4, s5
+; GFX942-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX942-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_cbranch_execnz .LBB27_1
; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8435,42 +8443,42 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-TRUE16-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-TRUE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_lshlrev_b32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_and_b32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v4.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v4, v2 offset:65532
+; GFX11-TRUE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v5, v3 offset:65532
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8483,39 +8491,39 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX11-FAKE16-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: ds_load_b32 v2, v0 offset:65532
+; GFX11-FAKE16-NEXT: ds_load_b32 v3, v0 offset:65532
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: s_mov_b32 s1, 0
; GFX11-FAKE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_sub_f32 v3, v4, v3 :: v_dual_and_b32 v6, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_sub_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v4, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v6, v4, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
; GFX11-FAKE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v5, v7, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v6, v8, s0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v3, v0, v3, v2 offset:65532
+; GFX11-FAKE16-NEXT: ds_cmpstore_rtn_b32 v4, v0, v4, v3 offset:65532
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v3
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, v4
; GFX11-FAKE16-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s1
@@ -8528,34 +8536,34 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX10-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX10-NEXT: ds_read_b32 v3, v0 offset:65532
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX10-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX10-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX10-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX10-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX10-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX10-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX10-NEXT: v_sub_f32_e32 v5, v5, v1
; GFX10-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX10-NEXT: v_bfe_u32 v7, v5, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX10-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX10-NEXT: v_cmp_u_f32_e64 s4, v3, v3
+; GFX10-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX10-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v7, s4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v6, v8, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v4, v3, 0x7060302
+; GFX10-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX10-NEXT: v_cmp_u_f32_e64 s4, v4, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v8, s4
+; GFX10-NEXT: v_perm_b32 v4, v5, v4, 0x7060302
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX10-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v2
-; GFX10-NEXT: v_mov_b32_e32 v2, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v3
+; GFX10-NEXT: v_mov_b32_e32 v3, v4
; GFX10-NEXT: s_or_b32 s5, vcc_lo, s5
; GFX10-NEXT: s_andn2_b32 exec_lo, exec_lo, s5
; GFX10-NEXT: s_cbranch_execnz .LBB27_1
@@ -8566,35 +8574,35 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX90A-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX90A: ; %bb.0:
; GFX90A-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX90A-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX90A-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX90A-NEXT: s_mov_b64 s[6:7], 0
+; GFX90A-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX90A-NEXT: s_movk_i32 s8, 0x7fff
+; GFX90A-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX90A-NEXT: s_mov_b32 s9, 0x7060302
; GFX90A-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX90A-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX90A-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX90A-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX90A-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX90A-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX90A-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX90A-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX90A-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX90A-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX90A-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX90A-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX90A-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX90A-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX90A-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX90A-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX90A-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX90A-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX90A-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX90A-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX90A-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX90A-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX90A-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX90A-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX90A-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX90A-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX90A-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX90A-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX90A-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX90A-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX90A-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX90A-NEXT: v_mov_b32_e32 v2, v3
+; GFX90A-NEXT: v_mov_b32_e32 v3, v4
; GFX90A-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX90A-NEXT: s_cbranch_execnz .LBB27_1
; GFX90A-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8604,35 +8612,35 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX908-LABEL: local_atomic_fsub_noret_v2bf16__ofset:
; GFX908: ; %bb.0:
; GFX908-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX908-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX908-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX908-NEXT: s_mov_b64 s[6:7], 0
+; GFX908-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX908-NEXT: s_movk_i32 s8, 0x7fff
+; GFX908-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX908-NEXT: s_mov_b32 s9, 0x7060302
; GFX908-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX908-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX908-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX908-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX908-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX908-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX908-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX908-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX908-NEXT: v_add3_u32 v5, v5, v3, s8
-; GFX908-NEXT: v_add3_u32 v7, v7, v4, s8
-; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX908-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX908-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX908-NEXT: v_perm_b32 v3, v4, v3, s9
-; GFX908-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX908-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX908-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX908-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX908-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX908-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX908-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX908-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX908-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX908-NEXT: v_add3_u32 v6, v6, v4, s8
+; GFX908-NEXT: v_add3_u32 v8, v8, v5, s8
+; GFX908-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX908-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX908-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX908-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX908-NEXT: v_perm_b32 v4, v5, v4, s9
+; GFX908-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX908-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX908-NEXT: v_mov_b32_e32 v2, v3
+; GFX908-NEXT: v_mov_b32_e32 v3, v4
; GFX908-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX908-NEXT: s_cbranch_execnz .LBB27_1
; GFX908-NEXT: ; %bb.2: ; %atomicrmw.end
@@ -8643,36 +8651,36 @@ define void @local_atomic_fsub_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_read_b32 v2, v0 offset:65532
+; GFX8-NEXT: ds_read_b32 v3, v0 offset:65532
; GFX8-NEXT: s_mov_b64 s[6:7], 0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX8-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX8-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX8-NEXT: v_sub_f32_e32 v3, v4, v3
-; GFX8-NEXT: v_sub_f32_e32 v4, v6, v5
-; GFX8-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX8-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v3
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v4
-; GFX8-NEXT: v_add_u32_e32 v5, vcc, 0x7fff, v5
-; GFX8-NEXT: v_add_u32_e32 v7, vcc, 0x7fff, v7
-; GFX8-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v4, v4
-; GFX8-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v3, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v6, s[4:5]
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_alignbit_b32 v3, v4, v3, 16
-; GFX8-NEXT: ds_cmpst_rtn_b32 v3, v0, v2, v3 offset:65532
+; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX8-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX8-NEXT: v_sub_f32_e32 v4, v4, v2
+; GFX8-NEXT: v_sub_f32_e32 v5, v5, v1
+; GFX8-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX8-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v4
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, v8, v5
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 0x7fff, v6
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 0x7fff, v8
+; GFX8-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX8-NEXT: v_cmp_u_f32_e32 vcc, v5, v5
+; GFX8-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX8-NEXT: v_cmp_u_f32_e64 s[4:5], v4, v4
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v7, s[4:5]
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_alignbit_b32 v4, v5, v4, 16
+; GFX8-NEXT: ds_cmpst_rtn_b32 v4, v0, v3, v4 offset:65532
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, v4, v3
; GFX8-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX8-NEXT: v_mov_b32_e32 v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, v4
; GFX8-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX8-NEXT: s_cbranch_execnz .LBB27_1
; GFX8-NEXT: ; %bb.2: ; %atomicrmw.end
diff --git a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
index 1b624decceed6..c442d22e3063a 100644
--- a/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-stack-alloc-block-sp-reference.ll
@@ -216,47 +216,46 @@ define amdgpu_kernel void @local_stack_offset_uses_sp(ptr addrspace(1) %out) #1
; FLATSCR-NEXT: s_mov_b32 s1, s0
; FLATSCR-NEXT: s_mov_b32 s2, s0
; FLATSCR-NEXT: s_mov_b32 s3, s0
-; FLATSCR-NEXT: s_mov_b32 s6, s0
-; FLATSCR-NEXT: .LBB0_1: ; %static-memset-expansion-main-body
-; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
-; FLATSCR-NEXT: s_add_i32 s7, s6, 0x3000
-; FLATSCR-NEXT: s_addk_i32 s6, 0x100
; FLATSCR-NEXT: v_mov_b32_e32 v0, s0
; FLATSCR-NEXT: v_mov_b32_e32 v1, s1
; FLATSCR-NEXT: v_mov_b32_e32 v2, s2
; FLATSCR-NEXT: v_mov_b32_e32 v3, s3
-; FLATSCR-NEXT: s_cmpk_lt_u32 s6, 0x2100
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:240
+; FLATSCR-NEXT: .LBB0_1: ; %static-memset-expansion-main-body
+; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
+; FLATSCR-NEXT: s_add_i32 s1, s0, 0x3000
+; FLATSCR-NEXT: s_addk_i32 s0, 0x100
+; FLATSCR-NEXT: s_cmpk_lt_u32 s0, 0x2100
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:240
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:224
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:224
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:208
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:208
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:192
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:192
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:176
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:176
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:160
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:160
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:144
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:144
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:128
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:128
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:112
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:112
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:96
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:96
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:80
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:80
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:64
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:64
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:48
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:48
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:32
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:32
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:16
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:16
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: s_cbranch_scc1 .LBB0_1
; FLATSCR-NEXT: ; %bb.2: ; %static-memset-post-expansion
@@ -506,10 +505,10 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) #1 {
; FLATSCR-LABEL: func_local_stack_offset_uses_sp:
; FLATSCR: ; %bb.0: ; %entry
; FLATSCR-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; FLATSCR-NEXT: s_mov_b32 s6, s33
+; FLATSCR-NEXT: s_mov_b32 s5, s33
; FLATSCR-NEXT: s_add_i32 s33, s32, 0x1fff
; FLATSCR-NEXT: s_and_b32 s33, s33, 0xffffe000
-; FLATSCR-NEXT: s_mov_b32 s7, s34
+; FLATSCR-NEXT: s_mov_b32 s6, s34
; FLATSCR-NEXT: s_mov_b32 s34, s32
; FLATSCR-NEXT: s_add_i32 s32, s32, 0x8000
; FLATSCR-NEXT: v_mov_b32_e32 v2, 0
@@ -520,48 +519,47 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) #1 {
; FLATSCR-NEXT: s_mov_b32 s1, s0
; FLATSCR-NEXT: s_mov_b32 s2, s0
; FLATSCR-NEXT: s_mov_b32 s3, s0
-; FLATSCR-NEXT: s_mov_b32 s4, s0
-; FLATSCR-NEXT: .LBB1_1: ; %static-memset-expansion-main-body
-; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
-; FLATSCR-NEXT: s_add_i32 s5, s33, s4
-; FLATSCR-NEXT: s_addk_i32 s5, 0x3000
-; FLATSCR-NEXT: s_addk_i32 s4, 0x100
; FLATSCR-NEXT: v_mov_b32_e32 v5, s3
; FLATSCR-NEXT: v_mov_b32_e32 v4, s2
; FLATSCR-NEXT: v_mov_b32_e32 v3, s1
; FLATSCR-NEXT: v_mov_b32_e32 v2, s0
-; FLATSCR-NEXT: s_cmpk_lt_u32 s4, 0x2100
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:240
+; FLATSCR-NEXT: .LBB1_1: ; %static-memset-expansion-main-body
+; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
+; FLATSCR-NEXT: s_add_i32 s1, s33, s0
+; FLATSCR-NEXT: s_addk_i32 s1, 0x3000
+; FLATSCR-NEXT: s_addk_i32 s0, 0x100
+; FLATSCR-NEXT: s_cmpk_lt_u32 s0, 0x2100
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:240
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:224
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:224
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:208
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:208
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:192
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:192
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:176
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:176
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:160
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:160
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:144
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:144
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:128
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:128
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:112
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:112
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:96
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:96
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:80
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:80
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:64
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:64
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:48
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:48
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:32
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:32
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5 offset:16
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1 offset:16
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s5
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[2:5], s1
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: s_cbranch_scc1 .LBB1_1
; FLATSCR-NEXT: ; %bb.2: ; %static-memset-post-expansion
@@ -586,8 +584,8 @@ define void @func_local_stack_offset_uses_sp(ptr addrspace(1) %out) #1 {
; FLATSCR-NEXT: scratch_load_dwordx2 v[4:5], off, s0 offset:64 glc
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: s_mov_b32 s32, s34
-; FLATSCR-NEXT: s_mov_b32 s34, s7
-; FLATSCR-NEXT: s_mov_b32 s33, s6
+; FLATSCR-NEXT: s_mov_b32 s34, s6
+; FLATSCR-NEXT: s_mov_b32 s33, s5
; FLATSCR-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4
; FLATSCR-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v5, vcc
; FLATSCR-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
@@ -848,47 +846,46 @@ define amdgpu_kernel void @local_stack_offset_uses_sp_flat(ptr addrspace(1) %out
; FLATSCR-NEXT: s_mov_b32 s1, s0
; FLATSCR-NEXT: s_mov_b32 s2, s0
; FLATSCR-NEXT: s_mov_b32 s3, s0
-; FLATSCR-NEXT: s_mov_b32 s6, s0
-; FLATSCR-NEXT: .LBB2_1: ; %static-memset-expansion-main-body
-; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
-; FLATSCR-NEXT: s_add_i32 s7, s6, 0x4000
-; FLATSCR-NEXT: s_addk_i32 s6, 0x100
; FLATSCR-NEXT: v_mov_b32_e32 v0, s0
; FLATSCR-NEXT: v_mov_b32_e32 v1, s1
; FLATSCR-NEXT: v_mov_b32_e32 v2, s2
; FLATSCR-NEXT: v_mov_b32_e32 v3, s3
-; FLATSCR-NEXT: s_cmpk_lt_u32 s6, 0x2100
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:240
+; FLATSCR-NEXT: .LBB2_1: ; %static-memset-expansion-main-body
+; FLATSCR-NEXT: ; =>This Inner Loop Header: Depth=1
+; FLATSCR-NEXT: s_add_i32 s1, s0, 0x4000
+; FLATSCR-NEXT: s_addk_i32 s0, 0x100
+; FLATSCR-NEXT: s_cmpk_lt_u32 s0, 0x2100
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:240
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:224
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:224
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:208
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:208
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:192
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:192
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:176
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:176
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:160
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:160
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:144
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:144
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:128
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:128
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:112
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:112
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:96
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:96
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:80
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:80
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:64
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:64
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:48
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:48
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:32
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:32
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7 offset:16
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1 offset:16
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
-; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s7
+; FLATSCR-NEXT: scratch_store_dwordx4 off, v[0:3], s1
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: s_cbranch_scc1 .LBB2_1
; FLATSCR-NEXT: ; %bb.2: ; %static-memset-post-expansion
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 4634ed0b6168e..d90c9771c3e4a 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -187,10 +187,10 @@ define void @issue63986_reduced_expanded(i64 %idxprom) {
; CHECK-NEXT: .LBB1_5: ; %loop-memcpy-residual.preheader
; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: s_mov_b32 s7, 0
-; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
-; CHECK-NEXT: s_add_i32 s6, s8, 1
; CHECK-NEXT: v_mov_b32_e32 v0, s4
; CHECK-NEXT: v_mov_b32_e32 v1, s5
+; CHECK-NEXT: .LBB1_6: ; %loop-memcpy-residual
+; CHECK-NEXT: s_add_i32 s6, s8, 1
; CHECK-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[0:1]
; CHECK-NEXT: s_mov_b64 s[8:9], 1
; CHECK-NEXT: s_cbranch_vccnz .LBB1_6
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index 61f79ccd55f1c..ad4d6b5a0f966 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -15964,24 +15964,25 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 s4, 0x41414141
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: s_mov_b32 s5, s4
; CHECK-NEXT: s_mov_b32 s6, s4
; CHECK-NEXT: s_mov_b32 s7, s4
-; CHECK-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
-; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v0, s8
-; CHECK-NEXT: s_add_u32 s8, s8, 0x100
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v1, vcc_lo
-; CHECK-NEXT: s_addc_u32 s9, s9, 0
-; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v6, 48
-; CHECK-NEXT: v_cmp_gt_u64_e64 s10, 0x800, s[8:9]
; CHECK-NEXT: v_mov_b32_e32 v2, s4
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mov_b32_e32 v4, s6
; CHECK-NEXT: v_mov_b32_e32 v5, s7
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_inst_prefetch 0x1
+; CHECK-NEXT: .p2align 6
+; CHECK-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v0, s4
+; CHECK-NEXT: s_add_u32 s4, s4, 0x100
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, s5, v1, vcc_lo
+; CHECK-NEXT: s_addc_u32 s5, s5, 0
+; CHECK-NEXT: v_add_co_u32 v8, vcc_lo, v6, 48
+; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v7, vcc_lo
-; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s10
; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[2:5] offset:128
; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[2:5] offset:64
; CHECK-NEXT: flat_store_dwordx4 v[6:7], v[2:5] offset:32
@@ -15998,8 +15999,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:48
; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:32
; CHECK-NEXT: flat_store_dwordx4 v[8:9], v[2:5]
+; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_vccnz .LBB10_1
; CHECK-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; CHECK-NEXT: s_inst_prefetch 0x2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
;
@@ -16348,26 +16351,26 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; UNROLL3: ; %bb.0: ; %entry
; UNROLL3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
-; UNROLL3-NEXT: s_mov_b64 s[8:9], 0
; UNROLL3-NEXT: s_mov_b32 s5, s4
; UNROLL3-NEXT: s_mov_b32 s6, s4
; UNROLL3-NEXT: s_mov_b32 s7, s4
-; UNROLL3-NEXT: .p2align 6
-; UNROLL3-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
-; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
-; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v0, s8
-; UNROLL3-NEXT: s_add_u32 s8, s8, 48
-; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v1, vcc_lo
-; UNROLL3-NEXT: s_addc_u32 s9, s9, 0
; UNROLL3-NEXT: v_mov_b32_e32 v2, s4
-; UNROLL3-NEXT: v_cmp_gt_u64_e64 s10, 0x7e0, s[8:9]
; UNROLL3-NEXT: v_mov_b32_e32 v3, s5
; UNROLL3-NEXT: v_mov_b32_e32 v4, s6
; UNROLL3-NEXT: v_mov_b32_e32 v5, s7
+; UNROLL3-NEXT: s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT: .p2align 6
+; UNROLL3-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
+; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v0, s4
+; UNROLL3-NEXT: s_add_u32 s4, s4, 48
+; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, s5, v1, vcc_lo
+; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[2:5] offset:16
; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[2:5]
-; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s10
+; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: flat_store_dwordx4 v[6:7], v[2:5] offset:32
+; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB10_1
; UNROLL3-NEXT: ; %bb.2: ; %static-memset-post-expansion
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
@@ -16392,23 +16395,22 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 s4, 0x41414141
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: s_mov_b32 s5, s4
; CHECK-NEXT: s_mov_b32 s6, s4
; CHECK-NEXT: s_mov_b32 s7, s4
-; CHECK-NEXT: s_inst_prefetch 0x1
-; CHECK-NEXT: .p2align 6
-; CHECK-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
-; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v0, s8
-; CHECK-NEXT: s_add_u32 s8, s8, 0x100
-; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v1, vcc_lo
-; CHECK-NEXT: s_addc_u32 s9, s9, 0
; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: v_cmp_gt_u64_e64 s10, 0x800, s[8:9]
; CHECK-NEXT: v_mov_b32_e32 v3, s5
; CHECK-NEXT: v_mov_b32_e32 v4, s6
; CHECK-NEXT: v_mov_b32_e32 v5, s7
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_inst_prefetch 0x1
+; CHECK-NEXT: .p2align 6
+; CHECK-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v0, s4
+; CHECK-NEXT: s_add_u32 s4, s4, 0x100
+; CHECK-NEXT: v_add_co_ci_u32_e64 v7, null, s5, v1, vcc_lo
+; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:240
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:224
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:208
@@ -16424,8 +16426,9 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:48
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:32
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:16
-; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s10
+; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: global_store_dwordx4 v[6:7], v[2:5], off
+; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_vccnz .LBB11_1
; CHECK-NEXT: ; %bb.2: ; %static-memset-post-expansion
; CHECK-NEXT: s_inst_prefetch 0x2
@@ -16773,26 +16776,26 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; UNROLL3: ; %bb.0: ; %entry
; UNROLL3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
-; UNROLL3-NEXT: s_mov_b64 s[8:9], 0
; UNROLL3-NEXT: s_mov_b32 s5, s4
; UNROLL3-NEXT: s_mov_b32 s6, s4
; UNROLL3-NEXT: s_mov_b32 s7, s4
-; UNROLL3-NEXT: .p2align 6
-; UNROLL3-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
-; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
-; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v0, s8
-; UNROLL3-NEXT: s_add_u32 s8, s8, 48
-; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v1, vcc_lo
-; UNROLL3-NEXT: s_addc_u32 s9, s9, 0
; UNROLL3-NEXT: v_mov_b32_e32 v2, s4
-; UNROLL3-NEXT: v_cmp_gt_u64_e64 s10, 0x7e0, s[8:9]
; UNROLL3-NEXT: v_mov_b32_e32 v3, s5
; UNROLL3-NEXT: v_mov_b32_e32 v4, s6
; UNROLL3-NEXT: v_mov_b32_e32 v5, s7
+; UNROLL3-NEXT: s_mov_b64 s[4:5], 0
+; UNROLL3-NEXT: .p2align 6
+; UNROLL3-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
+; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
+; UNROLL3-NEXT: v_add_co_u32 v6, vcc_lo, v0, s4
+; UNROLL3-NEXT: s_add_u32 s4, s4, 48
+; UNROLL3-NEXT: v_add_co_ci_u32_e64 v7, null, s5, v1, vcc_lo
+; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:16
; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[2:5], off
-; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s10
+; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:32
+; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB11_1
; UNROLL3-NEXT: ; %bb.2: ; %static-memset-post-expansion
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
@@ -16816,20 +16819,20 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) #1 {
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_mov_b32 s4, 0x41414141
-; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: s_mov_b32 s5, s4
; CHECK-NEXT: s_mov_b32 s6, s4
; CHECK-NEXT: s_mov_b32 s7, s4
-; CHECK-NEXT: s_inst_prefetch 0x1
-; CHECK-NEXT: .p2align 6
-; CHECK-NEXT: .LBB12_1: ; %static-memset-expansion-main-body
-; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_add_u32 s8, s8, 0x100
; CHECK-NEXT: v_mov_b32_e32 v1, s4
; CHECK-NEXT: v_mov_b32_e32 v2, s5
; CHECK-NEXT: v_mov_b32_e32 v3, s6
; CHECK-NEXT: v_mov_b32_e32 v4, s7
-; CHECK-NEXT: s_addc_u32 s9, s9, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_inst_prefetch 0x1
+; CHECK-NEXT: .p2align 6
+; CHECK-NEXT: .LBB12_1: ; %static-memset-expansion-main-body
+; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: s_add_u32 s4, s4, 0x100
+; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:240
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:224
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:208
@@ -16841,14 +16844,14 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) #1 {
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:112
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:96
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:80
+; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:64
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:48
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:32
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:16
; CHECK-NEXT: ds_write_b128 v0, v[1:4]
-; CHECK-NEXT: v_cmp_gt_u64_e64 s10, 0x800, s[8:9]
; CHECK-NEXT: v_add_nc_u32_e32 v0, 0x100, v0
-; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s10
+; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; CHECK-NEXT: s_cbranch_vccnz .LBB12_1
; CHECK-NEXT: ; %bb.2: ; %static-memset-post-expansion
; CHECK-NEXT: s_inst_prefetch 0x2
@@ -17131,27 +17134,26 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) #1 {
; UNROLL3-LABEL: memset_p3_sz2048:
; UNROLL3: ; %bb.0: ; %entry
; UNROLL3-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNROLL3-NEXT: v_mov_b32_e32 v1, v0
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
-; UNROLL3-NEXT: s_mov_b64 s[8:9], 0
+; UNROLL3-NEXT: v_mov_b32_e32 v5, v0
; UNROLL3-NEXT: s_mov_b32 s5, s4
; UNROLL3-NEXT: s_mov_b32 s6, s4
; UNROLL3-NEXT: s_mov_b32 s7, s4
-; UNROLL3-NEXT: .p2align 6
+; UNROLL3-NEXT: v_mov_b32_e32 v1, s4
+; UNROLL3-NEXT: v_mov_b32_e32 v2, s5
+; UNROLL3-NEXT: v_mov_b32_e32 v3, s6
+; UNROLL3-NEXT: v_mov_b32_e32 v4, s7
+; UNROLL3-NEXT: s_mov_b64 s[4:5], 0
; UNROLL3-NEXT: .LBB12_1: ; %static-memset-expansion-main-body
; UNROLL3-NEXT: ; =>This Inner Loop Header: Depth=1
-; UNROLL3-NEXT: s_add_u32 s8, s8, 48
-; UNROLL3-NEXT: s_addc_u32 s9, s9, 0
-; UNROLL3-NEXT: v_mov_b32_e32 v2, s4
-; UNROLL3-NEXT: v_mov_b32_e32 v3, s5
-; UNROLL3-NEXT: v_mov_b32_e32 v4, s6
-; UNROLL3-NEXT: v_mov_b32_e32 v5, s7
-; UNROLL3-NEXT: v_cmp_gt_u64_e64 s10, 0x7e0, s[8:9]
-; UNROLL3-NEXT: ds_write_b128 v1, v[2:5] offset:16
-; UNROLL3-NEXT: ds_write_b128 v1, v[2:5]
-; UNROLL3-NEXT: ds_write_b128 v1, v[2:5] offset:32
-; UNROLL3-NEXT: v_add_nc_u32_e32 v1, 48, v1
-; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s10
+; UNROLL3-NEXT: s_add_u32 s4, s4, 48
+; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
+; UNROLL3-NEXT: ds_write_b128 v5, v[1:4] offset:16
+; UNROLL3-NEXT: ds_write_b128 v5, v[1:4]
+; UNROLL3-NEXT: ds_write_b128 v5, v[1:4] offset:32
+; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
+; UNROLL3-NEXT: v_add_nc_u32_e32 v5, 48, v5
+; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB12_1
; UNROLL3-NEXT: ; %bb.2: ; %static-memset-post-expansion
; UNROLL3-NEXT: s_mov_b32 s4, 0x41414141
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1845bb56be388..46f654385cbc6 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -1703,40 +1703,40 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set40:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v6, vcc, v2, v4
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v7, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0x28282828
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0x28282828
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
-; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
-; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow4
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
@@ -1748,15 +1748,15 @@ define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %s
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, v0, v6
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v7, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow2
@@ -1825,40 +1825,40 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set0:
; GFX942-GISEL: ; %bb.0: ; %entry
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], 15
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v6, vcc, v2, v4
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v7, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0
-; GFX942-GISEL-NEXT: s_mov_b32 s1, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s2, s0
-; GFX942-GISEL-NEXT: s_mov_b32 s3, s0
-; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[4:5]
-; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
-; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
-; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow4
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
@@ -1870,15 +1870,15 @@ define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %si
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, v0, v6
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, v1, v7, vcc
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[4:5]
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX942-GISEL-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow2
diff --git a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
index 8395587d94185..1767f898f231e 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-pattern.ll
@@ -111,62 +111,61 @@ define void @memset_pattern_i128_constlen_mainloop_and_residual_taken(ptr addrsp
; GFX942-GISEL-LABEL: memset_pattern_i128_constlen_mainloop_and_residual_taken:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0xdddddddd
-; GFX942-GISEL-NEXT: s_mov_b32 s1, 0xcccccccc
-; GFX942-GISEL-NEXT: s_mov_b32 s2, 0xbbbbbbbb
-; GFX942-GISEL-NEXT: s_mov_b32 s3, 0xaaaaaaaa
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[0:1]
; GFX942-GISEL-NEXT: .LBB3_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:48
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:64
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:80
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:96
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:112
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:128
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:144
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:160
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:176
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:192
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:208
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:224
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[2:3], v[4:7], off offset:240
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v2, vcc, 0x1000, v2
-; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX942-GISEL-NEXT: s_xor_b32 s2, s2, 1
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:112
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:128
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:144
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:160
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:176
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:192
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:208
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:224
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[6:7], v[2:5], off offset:240
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v6, vcc, 0x1000, v6
+; GFX942-GISEL-NEXT: s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: s_cmp_lg_u32 s6, 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v7, vcc, 0, v7, vcc
+; GFX942-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB3_1
; GFX942-GISEL-NEXT: ; %bb.2: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x100, v0
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0xdddddddd
-; GFX942-GISEL-NEXT: s_mov_b32 s1, 0xcccccccc
-; GFX942-GISEL-NEXT: s_mov_b32 s2, 0xbbbbbbbb
-; GFX942-GISEL-NEXT: s_mov_b32 s3, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, 0x100, v0
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
; GFX942-GISEL-NEXT: .LBB3_3: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
-; GFX942-GISEL-NEXT: v_cmp_lt_u64_e64 s[6:7], s[4:5], 3
-; GFX942-GISEL-NEXT: s_cmp_lg_u64 s[6:7], 0
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, 16, v4
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e64 s[2:3], s[0:1], 3
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB3_3
; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -274,19 +273,19 @@ define void @memset_pattern_i128_len16(ptr addrspace(1) align 16 %a) {
; GFX942-GISEL-LABEL: memset_pattern_i128_len16:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0xdddddddd
-; GFX942-GISEL-NEXT: s_mov_b32 s1, 0xcccccccc
-; GFX942-GISEL-NEXT: s_mov_b32 s2, 0xbbbbbbbb
-; GFX942-GISEL-NEXT: s_mov_b32 s3, 0xaaaaaaaa
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
; GFX942-GISEL-NEXT: .LBB6_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
-; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-GISEL-NEXT: s_cselect_b32 s6, 1, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_cselect_b32 s2, 1, 0
+; GFX942-GISEL-NEXT: s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:16
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:32
@@ -304,10 +303,10 @@ define void @memset_pattern_i128_len16(ptr addrspace(1) align 16 %a) {
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:224
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[2:5], off offset:240
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 0x1000, v0
-; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_xor_b32 s2, s2, 1
; GFX942-GISEL-NEXT: s_nop 0
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX942-GISEL-NEXT: s_cmp_lg_u32 s6, 0
+; GFX942-GISEL-NEXT: s_cmp_lg_u32 s2, 0
; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB6_1
; GFX942-GISEL-NEXT: ; %bb.2: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -350,29 +349,29 @@ define void @memset_pattern_i128_dynlen(ptr addrspace(1) align 16 %a, i64 %len)
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB7_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
-; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
-; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
-; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
-; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s10, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s11, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT: .LBB7_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s10, s10, 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-GISEL-NEXT: s_addc_u32 s11, s11, 0
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 16, v0
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[10:11], v[2:3]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[2:3]
; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB7_2
; GFX942-GISEL-NEXT: .LBB7_3: ; %Flow1
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(1) %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
@@ -413,32 +412,32 @@ define void @memset_pattern_i128_dynlen_lds(ptr addrspace(3) align 16 %a, i64 %l
; GFX942-GISEL-LABEL: memset_pattern_i128_dynlen_lds:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB8_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s0, 0xdddddddd
-; GFX942-GISEL-NEXT: s_mov_b32 s1, 0xcccccccc
-; GFX942-GISEL-NEXT: s_mov_b32 s2, 0xbbbbbbbb
-; GFX942-GISEL-NEXT: s_mov_b32 s3, 0xaaaaaaaa
-; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xdddddddd
+; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xcccccccc
+; GFX942-GISEL-NEXT: s_mov_b32 s6, 0xbbbbbbbb
+; GFX942-GISEL-NEXT: s_mov_b32 s7, 0xaaaaaaaa
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: .LBB8_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 1
-; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[4:5]
-; GFX942-GISEL-NEXT: ds_write_b128 v0, v[6:9]
-; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-GISEL-NEXT: ds_write_b128 v0, v[2:5]
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 16, v0
-; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB8_2
; GFX942-GISEL-NEXT: .LBB8_3: ; %Flow1
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
call void @llvm.experimental.memset.pattern(ptr addrspace(3) align 16 %a, i128 u0xaaaaaaaabbbbbbbbccccccccdddddddd, i64 %len, i1 false)
@@ -504,39 +503,39 @@ define void @memset_pattern_i32_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
; GFX942-GISEL-LABEL: memset_pattern_i32_dynlen:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], 3
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 3, v2
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v6, vcc, v2, v4
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 3
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 3, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v7, vcc, 0, v3, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[8:9], vcc
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xaabbccdd
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
-; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
-; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], v[0:1]
-; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], 0
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xaabbccdd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s10, s8
+; GFX942-GISEL-NEXT: s_mov_b32 s11, s8
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
; GFX942-GISEL-NEXT: .LBB9_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s10, s10, 4
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[4:5]
-; GFX942-GISEL-NEXT: s_addc_u32 s11, s11, 0
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[10:13], off
-; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, 64, v8
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[10:11], v[6:7]
+; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 4
+; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, 64, v12
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[10:11]
; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, 0, v13, vcc
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB9_2
; GFX942-GISEL-NEXT: .LBB9_3: ; %Flow6
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_mov_b64 s[2:3], 0
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB9_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
@@ -550,7 +549,7 @@ define void @memset_pattern_i32_dynlen(ptr addrspace(1) align 16 %a, i64 %len) {
; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
; GFX942-GISEL-NEXT: global_store_dword v[0:1], v2, off
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, 4, v0
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[4:5]
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e64 s[0:1], s[6:7], v[8:9]
; GFX942-GISEL-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
@@ -799,57 +798,56 @@ define void @memset_pattern_i64_as7_dynlen(ptr addrspace(7) inreg align 16 %a, i
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_dynlen:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], 1
-; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 1, v4
-; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v6, vcc, v4, v2
-; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 1
+; GFX942-GISEL-NEXT: v_and_b32_e32 v6, 1, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v8, v6
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX942-GISEL-NEXT: s_nop 0
-; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v7, vcc, 0, v5, vcc
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[6:7], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: s_mov_b32 s4, 0xccccdddd
-; GFX942-GISEL-NEXT: s_mov_b32 s5, 0xaaaabbbb
-; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xccccdddd
+; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xaaaabbbb
+; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-GISEL-NEXT: s_mov_b64 s[12:13], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT: .LBB12_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s12, s12, 2
-; GFX942-GISEL-NEXT: s_addc_u32 s13, s13, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[6:7]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[4:5]
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[12:13], v[6:7]
-; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[8:11], v1, s[0:3], 0 offen
-; GFX942-GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 2
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[10:11]
+; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
-; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow3
-; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
; GFX942-GISEL-NEXT: ; %bb.4: ; %memset.pattern-expansion-residual-body.preheader
-; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 1, v[4:5]
-; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v4
-; GFX942-GISEL-NEXT: v_add3_u32 v0, v0, v1, s16
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 1, v[8:9]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v2
+; GFX942-GISEL-NEXT: v_add3_u32 v2, v0, v1, s16
; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xccccdddd
-; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xaaaabbbb
-; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], 0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-GISEL-NEXT: s_mov_b64 s[8:9], 0
; GFX942-GISEL-NEXT: .LBB12_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-GISEL-NEXT: s_add_u32 s10, s10, 1
-; GFX942-GISEL-NEXT: s_addc_u32 s11, s11, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[8:9]
-; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[10:11], v[2:3]
-; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[4:5], v0, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: s_add_u32 s8, s8, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s9, s9, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
; GFX942-GISEL-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
-; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v2, 8, v2
; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow1
@@ -1048,10 +1046,11 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-GISEL-LABEL: memset_pattern_i64_as7_len33_dynval:
; GFX942-GISEL: ; %bb.0:
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_add_u32_e32 v0, s16, v0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v1
; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, s16, v0
; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, v0
; GFX942-GISEL-NEXT: .LBB14_1: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -1059,7 +1058,6 @@ define void @memset_pattern_i64_as7_len33_dynval(ptr addrspace(7) inreg align 16
; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
; GFX942-GISEL-NEXT: s_cselect_b32 s6, 1, 0
; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], v[4:5]
; GFX942-GISEL-NEXT: s_xor_b32 s6, s6, 1
; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[4:7], v1, s[0:3], 0 offen offset:16
@@ -1093,24 +1091,25 @@ define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7
; GFX942-SDAG-LABEL: memset_pattern_i64_as7_dynlen_unaligned_uniform_len:
; GFX942-SDAG: ; %bb.0:
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-SDAG-NEXT: s_mov_b32 s7, s18
+; GFX942-SDAG-NEXT: s_mov_b32 s9, s18
; GFX942-SDAG-NEXT: s_and_b32 s4, s17, 1
-; GFX942-SDAG-NEXT: s_and_b32 s6, s17, -2
+; GFX942-SDAG-NEXT: s_and_b32 s8, s17, -2
; GFX942-SDAG-NEXT: s_mov_b32 s5, 0
-; GFX942-SDAG-NEXT: s_cmp_eq_u64 s[6:7], 0
-; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0
+; GFX942-SDAG-NEXT: s_cmp_eq_u64 s[8:9], 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0
; GFX942-SDAG-NEXT: s_cbranch_scc1 .LBB15_3
; GFX942-SDAG-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0xccccdddd
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0xccccdddd
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
; GFX942-SDAG-NEXT: v_add_u32_e32 v1, s16, v0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], v[4:5]
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[8:9]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
; GFX942-SDAG-NEXT: .LBB15_2: ; %memset.pattern-expansion-main-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-SDAG-NEXT: s_add_u32 s8, s8, 2
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
-; GFX942-SDAG-NEXT: s_addc_u32 s9, s9, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[6:7]
+; GFX942-SDAG-NEXT: s_add_u32 s6, s6, 2
+; GFX942-SDAG-NEXT: s_addc_u32 s7, s7, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
; GFX942-SDAG-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-SDAG-NEXT: v_add_u32_e32 v1, 32, v1
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB15_2
@@ -1121,18 +1120,18 @@ define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7
; GFX942-SDAG-NEXT: s_lshl_b32 s6, s17, 3
; GFX942-SDAG-NEXT: s_and_b32 s6, s6, -16
; GFX942-SDAG-NEXT: s_add_i32 s16, s16, s6
-; GFX942-SDAG-NEXT: v_add_u32_e32 v2, s16, v0
+; GFX942-SDAG-NEXT: v_add_u32_e32 v4, s16, v0
; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0xccccdddd
; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[4:5]
; GFX942-SDAG-NEXT: .LBB15_5: ; %memset.pattern-expansion-residual-body
; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-SDAG-NEXT: s_add_u32 s6, s6, 1
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: s_addc_u32 s7, s7, 0
-; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[4:5]
-; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
-; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 8, v2
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
+; GFX942-SDAG-NEXT: buffer_store_dwordx2 v[0:1], v4, s[0:3], 0 offen
+; GFX942-SDAG-NEXT: v_add_u32_e32 v4, 8, v4
; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB15_5
; GFX942-SDAG-NEXT: .LBB15_6: ; %memset.pattern-post-expansion
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
@@ -1150,17 +1149,17 @@ define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7
; GFX942-GISEL-NEXT: s_mov_b64 s[10:11], 0
; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_3
; GFX942-GISEL-NEXT: ; %bb.1: ; %memset.pattern-expansion-main-body.preheader
-; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0xccccdddd
; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 0xaaaabbbb
+; GFX942-GISEL-NEXT: v_add_u32_e32 v1, s16, v0
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v3
; GFX942-GISEL-NEXT: .LBB15_2: ; %memset.pattern-expansion-main-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: s_add_u32 s10, s10, 2
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[8:9]
; GFX942-GISEL-NEXT: s_addc_u32 s11, s11, 0
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[6:7]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[10:11]
+; GFX942-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, s[8:9], v[6:7]
; GFX942-GISEL-NEXT: buffer_store_dwordx4 v[2:5], v1, s[0:3], 0 offen
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; GFX942-GISEL-NEXT: v_add_u32_e32 v1, 32, v1
@@ -1173,20 +1172,19 @@ define void @memset_pattern_i64_as7_dynlen_unaligned_uniform_len(ptr addrspace(7
; GFX942-GISEL-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
; GFX942-GISEL-NEXT: s_lshl_b32 s6, s6, 4
; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, s16
-; GFX942-GISEL-NEXT: v_add3_u32 v0, v0, s6, v1
+; GFX942-GISEL-NEXT: v_add3_u32 v2, v0, s6, v1
; GFX942-GISEL-NEXT: s_mov_b64 s[6:7], 0
-; GFX942-GISEL-NEXT: s_mov_b32 s8, 0xccccdddd
-; GFX942-GISEL-NEXT: s_mov_b32 s9, 0xaaaabbbb
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0xccccdddd
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v1, 0xaaaabbbb
; GFX942-GISEL-NEXT: .LBB15_5: ; %memset.pattern-expansion-residual-body
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: s_add_u32 s6, s6, 1
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: s_addc_u32 s7, s7, 0
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[8:9]
-; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[4:5]
-; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[2:3], v0, s[0:3], 0 offen
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[6:7]
+; GFX942-GISEL-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-GISEL-NEXT: buffer_store_dwordx2 v[0:1], v2, s[0:3], 0 offen
; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
-; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 8, v0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v2, 8, v2
; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB15_5
; GFX942-GISEL-NEXT: .LBB15_6: ; %memset.pattern-post-expansion
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
index 43409292fec9b..61f070ec53105 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-scavenge-fi-stack-id.ll
@@ -6,10 +6,10 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
+; CHECK-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
-; CHECK-NEXT: v_writelane_b32 v12, s30, 0
-; CHECK-NEXT: v_writelane_b32 v12, s31, 1
+; CHECK-NEXT: v_writelane_b32 v10, s30, 0
+; CHECK-NEXT: v_writelane_b32 v10, s31, 1
; CHECK-NEXT: v_and_b32_e32 v2, 1, v2
; CHECK-NEXT: v_cmp_eq_u32_e64 s[10:11], 1, v2
; CHECK-NEXT: v_and_b32_e32 v2, 1, v4
@@ -138,10 +138,15 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_cbranch_execz .LBB0_14
; CHECK-NEXT: ; %bb.12: ; %inner_stack_loop.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_11 Depth=4
-; CHECK-NEXT: s_mov_b32 vcc_lo, 0
; CHECK-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
; CHECK-NEXT: v_cndmask_b32_e64 v4, v2, 0, s[4:5]
+; CHECK-NEXT: v_cndmask_b32_e64 v5, v5, 0, s[14:15]
+; CHECK-NEXT: v_cndmask_b32_e64 v4, v4, 0, s[14:15]
+; CHECK-NEXT: v_mul_f64 v[4:5], v[4:5], s[28:29]
+; CHECK-NEXT: v_mul_f64 v[4:5], v[4:5], s[40:41]
+; CHECK-NEXT: s_mov_b32 vcc_lo, 0
; CHECK-NEXT: s_mov_b64 s[94:95], 0
+; CHECK-NEXT: v_mul_f64 v[4:5], v[4:5], s[42:43]
; CHECK-NEXT: .LBB0_13: ; %inner_stack_loop
; CHECK-NEXT: ; Parent Loop BB0_2 Depth=1
; CHECK-NEXT: ; Parent Loop BB0_4 Depth=2
@@ -176,15 +181,10 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: v_mov_b32_e32 v7, vcc_hi
; CHECK-NEXT: buffer_load_dword v8, v7, s[0:3], 0 offen
; CHECK-NEXT: buffer_load_dword v9, v7, s[0:3], 0 offen offset:4
-; CHECK-NEXT: v_cndmask_b32_e64 v11, v5, 0, s[14:15]
-; CHECK-NEXT: v_cndmask_b32_e64 v10, v4, 0, s[14:15]
-; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[28:29]
-; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[40:41]
-; CHECK-NEXT: v_mul_f64 v[10:11], v[10:11], s[42:43]
; CHECK-NEXT: v_mov_b32_e32 v7, vcc_lo
; CHECK-NEXT: s_mov_b32 vcc_lo, 1
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_mul_f64 v[8:9], v[10:11], v[8:9]
+; CHECK-NEXT: v_mul_f64 v[8:9], v[4:5], v[8:9]
; CHECK-NEXT: buffer_store_dword v9, off, s[0:3], 0 offset:4
; CHECK-NEXT: buffer_store_dword v8, off, s[0:3], 0
; CHECK-NEXT: buffer_store_dword v6, v7, s[0:3], 0 offen
@@ -232,10 +232,10 @@ define void @sgpr_scavenge_fi_stack_id(double %input, i1 %enter_fma_path, i1 %re
; CHECK-NEXT: s_branch .LBB0_3
; CHECK-NEXT: .LBB0_21: ; %DummyReturnBlock
; CHECK-NEXT: s_or_b64 exec, exec, s[20:21]
-; CHECK-NEXT: v_readlane_b32 s30, v12, 0
-; CHECK-NEXT: v_readlane_b32 s31, v12, 1
+; CHECK-NEXT: v_readlane_b32 s30, v10, 0
+; CHECK-NEXT: v_readlane_b32 s31, v10, 1
; CHECK-NEXT: s_xor_saveexec_b64 s[4:5], -1
-; CHECK-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; CHECK-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
; CHECK-NEXT: s_mov_b64 exec, s[4:5]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
index c1ce87b3ef54c..981dd9a81243a 100644
--- a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
+++ b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
@@ -85,67 +85,62 @@ merge:
define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900-LABEL: test_loop_with_if:
; GFX900: ; %bb.0: ; %entry
-; GFX900-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: v_mov_b32_e32 v3, 0
-; GFX900-NEXT: s_mov_b64 s[2:3], 0
-; GFX900-NEXT: s_movk_i32 s12, 0xfe
+; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX900-NEXT: v_mov_b32_e32 v5, 0
+; GFX900-NEXT: s_mov_b64 s[0:1], 0
+; GFX900-NEXT: s_movk_i32 s10, 0xfe
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_bitcmp1_b32 s6, 0
-; GFX900-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX900-NEXT: s_xor_b64 s[6:7], s[4:5], -1
+; GFX900-NEXT: s_bitcmp1_b32 s2, 0
+; GFX900-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX900-NEXT: s_xor_b64 s[4:5], s[2:3], -1
+; GFX900-NEXT: v_mov_b32_e32 v1, s6
+; GFX900-NEXT: v_mov_b32_e32 v2, s7
; GFX900-NEXT: s_branch .LBB2_2
; GFX900-NEXT: .LBB2_1: ; %latch
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_or_b64 exec, exec, s[10:11]
+; GFX900-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_u32_e32 v3, 20, v1
-; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s12, v3
-; GFX900-NEXT: v_mov_b32_e32 v5, s1
-; GFX900-NEXT: v_mov_b32_e32 v4, s0
-; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX900-NEXT: flat_store_dword v[4:5], v1
-; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX900-NEXT: v_add_u32_e32 v5, 20, v3
+; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s10, v5
+; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX900-NEXT: flat_store_dword v[1:2], v3
+; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execz .LBB2_8
; GFX900-NEXT: .LBB2_2: ; %loop
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX900-NEXT: v_mov_b32_e32 v2, s1
-; GFX900-NEXT: v_mov_b32_e32 v1, s0
-; GFX900-NEXT: flat_load_dwordx2 v[1:2], v[1:2]
-; GFX900-NEXT: s_and_b64 s[8:9], s[4:5], exec
-; GFX900-NEXT: s_cselect_b32 s8, 1, 0
-; GFX900-NEXT: s_cmp_lg_u32 s8, 1
-; GFX900-NEXT: s_mov_b64 s[10:11], s[6:7]
-; GFX900-NEXT: s_mov_b64 s[8:9], 0
+; GFX900-NEXT: flat_load_dwordx2 v[3:4], v[1:2]
+; GFX900-NEXT: s_and_b64 s[6:7], s[2:3], exec
+; GFX900-NEXT: s_cselect_b32 s6, 1, 0
+; GFX900-NEXT: s_cmp_lg_u32 s6, 1
+; GFX900-NEXT: s_mov_b64 s[8:9], s[4:5]
+; GFX900-NEXT: s_mov_b64 s[6:7], 0
; GFX900-NEXT: s_cbranch_scc1 .LBB2_4
; GFX900-NEXT: ; %bb.3: ; %if
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v3
-; GFX900-NEXT: s_andn2_b64 s[10:11], s[6:7], exec
-; GFX900-NEXT: s_and_b64 s[14:15], vcc, exec
-; GFX900-NEXT: s_mov_b64 s[8:9], -1
-; GFX900-NEXT: s_or_b64 s[10:11], s[10:11], s[14:15]
+; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v5
+; GFX900-NEXT: s_andn2_b64 s[8:9], s[4:5], exec
+; GFX900-NEXT: s_and_b64 s[12:13], vcc, exec
+; GFX900-NEXT: s_mov_b64 s[6:7], -1
+; GFX900-NEXT: s_or_b64 s[8:9], s[8:9], s[12:13]
; GFX900-NEXT: .LBB2_4: ; %Flow
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_and_saveexec_b64 s[14:15], s[10:11]
-; GFX900-NEXT: s_xor_b64 s[10:11], exec, s[14:15]
+; GFX900-NEXT: s_and_saveexec_b64 s[12:13], s[8:9]
+; GFX900-NEXT: s_xor_b64 s[8:9], exec, s[12:13]
; GFX900-NEXT: s_cbranch_execz .LBB2_6
; GFX900-NEXT: ; %bb.5: ; %else
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_add_u32_e32 v1, v1, v2
-; GFX900-NEXT: s_andn2_b64 s[8:9], s[8:9], exec
+; GFX900-NEXT: v_add_u32_e32 v3, v3, v4
+; GFX900-NEXT: s_andn2_b64 s[6:7], s[6:7], exec
; GFX900-NEXT: .LBB2_6: ; %Flow1
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX900-NEXT: s_and_saveexec_b64 s[10:11], s[8:9]
+; GFX900-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX900-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
; GFX900-NEXT: s_cbranch_execz .LBB2_1
; GFX900-NEXT: ; %bb.7: ; %then
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
-; GFX900-NEXT: v_mov_b32_e32 v3, s1
-; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s0
-; GFX900-NEXT: flat_store_dword v[2:3], v0
+; GFX900-NEXT: flat_store_dword v[1:2], v0
; GFX900-NEXT: s_branch .LBB2_1
; GFX900-NEXT: .LBB2_8: ; %end
; GFX900-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/valu-i1.ll b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
index aa38b29b0bf2a..0f934e7ff2303 100644
--- a/llvm/test/CodeGen/AMDGPU/valu-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/valu-i1.ll
@@ -324,12 +324,12 @@ define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %
; SI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; SI-NEXT: s_mov_b64 s[2:3], 0
+; SI-NEXT: s_mov_b32 s8, s10
+; SI-NEXT: s_mov_b32 s9, s10
; SI-NEXT: s_mov_b64 s[6:7], 0
; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
; SI-NEXT: .LBB5_2: ; %bb10
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_mov_b32 s8, s10
-; SI-NEXT: s_mov_b32 s9, s10
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: buffer_load_dword v8, v[6:7], s[8:11], 0 addr64
; SI-NEXT: buffer_load_dword v9, v[4:5], s[8:11], 0 addr64
@@ -355,8 +355,8 @@ define amdgpu_kernel void @multi_vcond_loop(ptr addrspace(1) noalias nocapture %
; SI-NEXT: v_addc_u32_e32 v7, vcc, 0, v7, vcc
; SI-NEXT: v_cmp_ge_i64_e32 vcc, s[6:7], v[0:1]
; SI-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
-; SI-NEXT: s_and_b64 s[8:9], vcc, exec
-; SI-NEXT: s_or_b64 s[4:5], s[4:5], s[8:9]
+; SI-NEXT: s_and_b64 s[12:13], vcc, exec
+; SI-NEXT: s_or_b64 s[4:5], s[4:5], s[12:13]
; SI-NEXT: .LBB5_4: ; %Flow
; SI-NEXT: ; in Loop: Header=BB5_2 Depth=1
; SI-NEXT: s_or_b64 exec, exec, s[0:1]
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index 36bb002a6fa4f..ccb0349bff19b 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -95,13 +95,13 @@
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: Machine Block Frequency Analysis
-; CHECK-NEXT: Machine Register Class Info Analysis
; CHECK-NEXT: Early Machine Loop Invariant Code Motion
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Block Frequency Analysis
; CHECK-NEXT: Machine Common Subexpression Elimination
; CHECK-NEXT: MachinePostDominator Tree Construction
; CHECK-NEXT: Machine Cycle Info Analysis
+; CHECK-NEXT: Machine Register Class Info Analysis
; CHECK-NEXT: Machine code sinking
; CHECK-NEXT: Peephole Optimizations
; CHECK-NEXT: Remove dead machine instructions
diff --git a/llvm/test/CodeGen/LoongArch/jr-without-ra.ll b/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
index 96c9e7523f642..1a1fe0e2b19e2 100644
--- a/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
+++ b/llvm/test/CodeGen/LoongArch/jr-without-ra.ll
@@ -20,101 +20,101 @@ define void @jr_without_ra(ptr %rtwdev, ptr %chan, ptr %h2c, i8 %.pre, i1 %cmp.i
; CHECK-NEXT: st.d $s6, $sp, 24 # 8-byte Folded Spill
; CHECK-NEXT: st.d $s7, $sp, 16 # 8-byte Folded Spill
; CHECK-NEXT: st.d $s8, $sp, 8 # 8-byte Folded Spill
-; CHECK-NEXT: move $s6, $zero
-; CHECK-NEXT: move $s1, $zero
+; CHECK-NEXT: move $s7, $zero
+; CHECK-NEXT: move $s0, $zero
; CHECK-NEXT: ld.d $t0, $sp, 184
-; CHECK-NEXT: ld.d $t1, $sp, 176
-; CHECK-NEXT: ld.d $s2, $sp, 168
-; CHECK-NEXT: ld.d $t2, $sp, 160
-; CHECK-NEXT: ld.d $t3, $sp, 152
-; CHECK-NEXT: ld.d $t4, $sp, 144
-; CHECK-NEXT: ld.d $t5, $sp, 136
-; CHECK-NEXT: ld.d $t6, $sp, 128
-; CHECK-NEXT: ld.d $t7, $sp, 120
-; CHECK-NEXT: ld.d $t8, $sp, 112
-; CHECK-NEXT: ld.d $fp, $sp, 104
-; CHECK-NEXT: ld.d $s0, $sp, 96
+; CHECK-NEXT: ld.d $s2, $sp, 176
+; CHECK-NEXT: ld.d $s1, $sp, 168
+; CHECK-NEXT: ld.d $t1, $sp, 160
+; CHECK-NEXT: ld.d $t2, $sp, 152
+; CHECK-NEXT: ld.d $t3, $sp, 144
+; CHECK-NEXT: ld.d $t4, $sp, 136
+; CHECK-NEXT: ld.d $t5, $sp, 128
+; CHECK-NEXT: ld.d $t6, $sp, 120
+; CHECK-NEXT: ld.d $t7, $sp, 112
+; CHECK-NEXT: ld.d $t8, $sp, 104
+; CHECK-NEXT: ld.d $fp, $sp, 96
; CHECK-NEXT: andi $a4, $a4, 1
-; CHECK-NEXT: alsl.d $a6, $a6, $s2, 4
-; CHECK-NEXT: pcalau12i $s2, %pc_hi20(.LJTI0_0)
-; CHECK-NEXT: addi.d $s2, $s2, %pc_lo12(.LJTI0_0)
+; CHECK-NEXT: alsl.d $a6, $a6, $s1, 4
+; CHECK-NEXT: pcalau12i $s1, %pc_hi20(.LJTI0_0)
+; CHECK-NEXT: addi.d $s1, $s1, %pc_lo12(.LJTI0_0)
+; CHECK-NEXT: slli.d $s3, $s2, 2
+; CHECK-NEXT: alsl.d $s2, $s2, $s3, 1
+; CHECK-NEXT: add.d $s2, $t5, $s2
+; CHECK-NEXT: addi.w $s4, $zero, -41
; CHECK-NEXT: ori $s3, $zero, 1
-; CHECK-NEXT: ori $s4, $zero, 50
-; CHECK-NEXT: ori $s5, $zero, 3
-; CHECK-NEXT: lu32i.d $s5, 262144
+; CHECK-NEXT: slli.d $s4, $s4, 3
+; CHECK-NEXT: ori $s6, $zero, 3
+; CHECK-NEXT: lu32i.d $s6, 262144
; CHECK-NEXT: b .LBB0_4
; CHECK-NEXT: .p2align 4, , 16
; CHECK-NEXT: .LBB0_1: # %sw.bb27.i.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: ori $s7, $zero, 1
+; CHECK-NEXT: ori $s8, $zero, 1
; CHECK-NEXT: .LBB0_2: # %if.else.i106
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: alsl.d $s8, $s1, $s1, 3
-; CHECK-NEXT: alsl.d $s1, $s8, $s1, 1
-; CHECK-NEXT: add.d $s1, $t0, $s1
-; CHECK-NEXT: ldx.bu $s7, $s1, $s7
+; CHECK-NEXT: alsl.d $s5, $s0, $s0, 3
+; CHECK-NEXT: alsl.d $s0, $s5, $s0, 1
+; CHECK-NEXT: add.d $s0, $t0, $s0
+; CHECK-NEXT: ldx.bu $s8, $s0, $s8
; CHECK-NEXT: .LBB0_3: # %phy_tssi_get_ofdm_de.exit
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: st.b $zero, $t6, 0
-; CHECK-NEXT: st.b $s6, $t4, 0
-; CHECK-NEXT: st.b $zero, $fp, 0
-; CHECK-NEXT: st.b $zero, $t2, 0
+; CHECK-NEXT: st.b $zero, $t5, 0
+; CHECK-NEXT: st.b $s7, $t3, 0
+; CHECK-NEXT: st.b $zero, $t8, 0
+; CHECK-NEXT: st.b $zero, $t1, 0
; CHECK-NEXT: st.b $zero, $a1, 0
-; CHECK-NEXT: st.b $zero, $t3, 0
-; CHECK-NEXT: st.b $s7, $a5, 0
-; CHECK-NEXT: ori $s1, $zero, 1
-; CHECK-NEXT: move $s6, $a3
+; CHECK-NEXT: st.b $zero, $t2, 0
+; CHECK-NEXT: st.b $s8, $a5, 0
+; CHECK-NEXT: ori $s0, $zero, 1
+; CHECK-NEXT: move $s7, $a3
; CHECK-NEXT: .LBB0_4: # %for.body
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
; CHECK-NEXT: beqz $a4, .LBB0_9
; CHECK-NEXT: # %bb.5: # %calc_6g.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: move $s6, $zero
+; CHECK-NEXT: move $s7, $zero
; CHECK-NEXT: bnez $zero, .LBB0_8
; CHECK-NEXT: # %bb.6: # %calc_6g.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: slli.d $s7, $zero, 3
-; CHECK-NEXT: ldx.d $s7, $s2, $s7
-; CHECK-NEXT: jr $s7
+; CHECK-NEXT: slli.d $s8, $zero, 3
+; CHECK-NEXT: ldx.d $s8, $s1, $s8
+; CHECK-NEXT: jr $s8
; CHECK-NEXT: .LBB0_7: # %sw.bb12.i.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: ori $s6, $zero, 1
+; CHECK-NEXT: ori $s7, $zero, 1
; CHECK-NEXT: .LBB0_8: # %if.else58.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: ldx.bu $s6, $a6, $s6
+; CHECK-NEXT: ldx.bu $s7, $a6, $s7
; CHECK-NEXT: b .LBB0_11
; CHECK-NEXT: .p2align 4, , 16
; CHECK-NEXT: .LBB0_9: # %if.end.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: andi $s6, $s6, 255
-; CHECK-NEXT: bltu $s4, $s6, .LBB0_15
+; CHECK-NEXT: andi $s7, $s7, 255
+; CHECK-NEXT: ori $s5, $zero, 50
+; CHECK-NEXT: bltu $s5, $s7, .LBB0_15
; CHECK-NEXT: # %bb.10: # %if.end.i
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: sll.d $s6, $s3, $s6
-; CHECK-NEXT: and $s7, $s6, $s5
-; CHECK-NEXT: move $s6, $s0
-; CHECK-NEXT: beqz $s7, .LBB0_15
+; CHECK-NEXT: sll.d $s7, $s3, $s7
+; CHECK-NEXT: and $s8, $s7, $s6
+; CHECK-NEXT: move $s7, $fp
+; CHECK-NEXT: beqz $s8, .LBB0_15
; CHECK-NEXT: .LBB0_11: # %phy_tssi_get_ofdm_trim_de.exit
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: move $s7, $zero
-; CHECK-NEXT: st.b $zero, $t8, 0
-; CHECK-NEXT: slli.d $s8, $t1, 2
-; CHECK-NEXT: alsl.d $s8, $t1, $s8, 1
-; CHECK-NEXT: add.d $s8, $t6, $s8
-; CHECK-NEXT: ldx.b $s8, $s8, $t5
+; CHECK-NEXT: move $s8, $zero
+; CHECK-NEXT: st.b $zero, $t7, 0
+; CHECK-NEXT: ldx.b $ra, $s2, $t4
; CHECK-NEXT: st.b $zero, $a2, 0
; CHECK-NEXT: st.b $zero, $a7, 0
-; CHECK-NEXT: st.b $zero, $t7, 0
-; CHECK-NEXT: st.b $s8, $a0, 0
+; CHECK-NEXT: st.b $zero, $t6, 0
+; CHECK-NEXT: st.b $ra, $a0, 0
; CHECK-NEXT: bnez $s3, .LBB0_13
; CHECK-NEXT: # %bb.12: # %phy_tssi_get_ofdm_trim_de.exit
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
-; CHECK-NEXT: addi.w $s8, $zero, -41
-; CHECK-NEXT: slli.d $s8, $s8, 3
; CHECK-NEXT: pcalau12i $ra, %pc_hi20(.LJTI0_1)
; CHECK-NEXT: addi.d $ra, $ra, %pc_lo12(.LJTI0_1)
-; CHECK-NEXT: ldx.d $s8, $ra, $s8
-; CHECK-NEXT: jr $s8
+; CHECK-NEXT: ldx.d $s5, $ra, $s4
+; CHECK-NEXT: jr $s5
; CHECK-NEXT: .LBB0_13: # %phy_tssi_get_ofdm_trim_de.exit
; CHECK-NEXT: # in Loop: Header=BB0_4 Depth=1
; CHECK-NEXT: bnez $s3, .LBB0_1
diff --git a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
index f65df23d5b134..96cb7850d1870 100644
--- a/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
+++ b/llvm/test/CodeGen/LoongArch/opt-pipeline.ll
@@ -101,13 +101,13 @@
; LAXX-NEXT: MachineDominator Tree Construction
; LAXX-NEXT: Machine Natural Loop Construction
; LAXX-NEXT: Machine Block Frequency Analysis
-; LAXX-NEXT: Machine Register Class Info Analysis
; LAXX-NEXT: Early Machine Loop Invariant Code Motion
; LAXX-NEXT: MachineDominator Tree Construction
; LAXX-NEXT: Machine Block Frequency Analysis
; LAXX-NEXT: Machine Common Subexpression Elimination
; LAXX-NEXT: MachinePostDominator Tree Construction
; LAXX-NEXT: Machine Cycle Info Analysis
+; LAXX-NEXT: Machine Register Class Info Analysis
; LAXX-NEXT: Machine code sinking
; LAXX-NEXT: Peephole Optimizations
; LAXX-NEXT: Remove dead machine instructions
diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
index 86018b99f5d10..da39af9ba8fb6 100644
--- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll
@@ -113,7 +113,6 @@
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
; CHECK-NEXT: Machine Block Frequency Analysis
-; CHECK-NEXT: Machine Register Class Info Analysis
; CHECK-NEXT: Early Machine Loop Invariant Code Motion
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: RISC-V VL Optimizer
@@ -128,6 +127,7 @@
; CHECK-NEXT: Remove dead machine instructions
; CHECK-NEXT: MachineDominator Tree Construction
; CHECK-NEXT: Machine Natural Loop Construction
+; CHECK-NEXT: Machine Register Class Info Analysis
; CHECK-NEXT: Machine Trace Metrics
; CHECK-NEXT: Lazy Machine Block Frequency Analysis
; CHECK-NEXT: Machine InstCombiner
diff --git a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
index 8069f53113497..cba670e62a79a 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vxrm-insert-out-of-loop.ll
@@ -489,9 +489,8 @@ define void @test1(ptr nocapture noundef writeonly %dst, i32 noundef signext %i_
; RV64-NEXT: j .LBB0_11
; RV64-NEXT: .LBB0_8: # %vector.ph
; RV64-NEXT: # in Loop: Header=BB0_6 Depth=1
-; RV64-NEXT: slli t6, t0, 1
-; RV64-NEXT: slli s0, t0, 28
-; RV64-NEXT: sub t6, s0, t6
+; RV64-NEXT: slli t6, t0, 28
+; RV64-NEXT: sub t6, t6, t1
; RV64-NEXT: and t6, t6, a6
; RV64-NEXT: mv s0, a2
; RV64-NEXT: mv s1, a4
diff --git a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
index ce9bc3c763fb1..287c25e97a481 100644
--- a/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
+++ b/llvm/test/CodeGen/SPIRV/llc-pipeline.ll
@@ -202,13 +202,13 @@
; SPIRV-Opt-NEXT: MachineDominator Tree Construction
; SPIRV-Opt-NEXT: Machine Natural Loop Construction
; SPIRV-Opt-NEXT: Machine Block Frequency Analysis
-; SPIRV-Opt-NEXT: Machine Register Class Info Analysis
; SPIRV-Opt-NEXT: Early Machine Loop Invariant Code Motion
; SPIRV-Opt-NEXT: MachineDominator Tree Construction
; SPIRV-Opt-NEXT: Machine Block Frequency Analysis
; SPIRV-Opt-NEXT: Machine Common Subexpression Elimination
; SPIRV-Opt-NEXT: MachinePostDominator Tree Construction
; SPIRV-Opt-NEXT: Machine Cycle Info Analysis
+; SPIRV-Opt-NEXT: Machine Register Class Info Analysis
; SPIRV-Opt-NEXT: Machine code sinking
; SPIRV-Opt-NEXT: Peephole Optimizations
; SPIRV-Opt-NEXT: Remove dead machine instructions
diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
index 5423a2bf75973..706a7c34c3df5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll
@@ -350,8 +350,8 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: .pad #4
; CHECK-NEXT: sub sp, #4
-; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: .pad #16
; CHECK-NEXT: sub sp, #16
; CHECK-NEXT: mov lr, r0
@@ -361,48 +361,50 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: @ %bb.1: @ %for.cond2.preheader.lr.ph
; CHECK-NEXT: movs r0, #1
; CHECK-NEXT: cmp r2, #1
-; CHECK-NEXT: csel r3, r2, r0, lt
+; CHECK-NEXT: csel r7, r2, r0, lt
; CHECK-NEXT: mov r12, r1
-; CHECK-NEXT: mov r1, r3
-; CHECK-NEXT: cmp r3, #3
+; CHECK-NEXT: mov r1, r7
+; CHECK-NEXT: cmp r7, #3
; CHECK-NEXT: it ls
; CHECK-NEXT: movls r1, #3
; CHECK-NEXT: mov r4, r2
-; CHECK-NEXT: subs r1, r1, r3
+; CHECK-NEXT: subs r1, r1, r7
; CHECK-NEXT: movw r2, #43691
; CHECK-NEXT: adds r1, #2
; CHECK-NEXT: movt r2, #43690
-; CHECK-NEXT: ldr r6, [sp, #112]
-; CHECK-NEXT: movw r9, :lower16:c
+; CHECK-NEXT: ldr r6, [sp, #128]
+; CHECK-NEXT: movw r8, :lower16:c
; CHECK-NEXT: umull r1, r2, r1, r2
-; CHECK-NEXT: adr.w r8, .LCPI1_1
+; CHECK-NEXT: movt r8, :upper16:c
; CHECK-NEXT: movs r1, #4
+; CHECK-NEXT: @ implicit-def: $r10
; CHECK-NEXT: @ implicit-def: $r5
; CHECK-NEXT: @ implicit-def: $r11
-; CHECK-NEXT: @ implicit-def: $r7
-; CHECK-NEXT: movt r9, :upper16:c
-; CHECK-NEXT: mov.w r10, #12
-; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill
+; CHECK-NEXT: mov.w r9, #12
+; CHECK-NEXT: str r4, [sp, #12] @ 4-byte Spill
; CHECK-NEXT: add.w r1, r1, r2, lsr #1
; CHECK-NEXT: add.w r0, r0, r2, lsr #1
-; CHECK-NEXT: bic r2, r1, #3
+; CHECK-NEXT: bic r3, r1, #3
; CHECK-NEXT: adr r1, .LCPI1_0
; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: vdup.32 q5, r0
+; CHECK-NEXT: adr r1, .LCPI1_1
+; CHECK-NEXT: vldrw.u32 q5, [r1]
; CHECK-NEXT: vdup.32 q6, r0
-; CHECK-NEXT: strd r2, r4, [sp, #4] @ 8-byte Folded Spill
-; CHECK-NEXT: vadd.i32 q4, q0, r3
+; CHECK-NEXT: vadd.i32 q4, q0, r7
+; CHECK-NEXT: vdup.32 q7, r0
+; CHECK-NEXT: strd r3, r7, [sp, #4] @ 8-byte Folded Spill
; CHECK-NEXT: b .LBB1_6
; CHECK-NEXT: .LBB1_2: @ %for.body6.preheader
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: mov r0, r7
-; CHECK-NEXT: cmn.w r7, #4
+; CHECK-NEXT: mov r0, r11
+; CHECK-NEXT: cmn.w r11, #4
; CHECK-NEXT: it le
; CHECK-NEXT: mvnle r0, #3
; CHECK-NEXT: movw r2, #18725
; CHECK-NEXT: adds r0, #6
; CHECK-NEXT: movt r2, #9362
-; CHECK-NEXT: subs r1, r0, r7
+; CHECK-NEXT: sub.w r1, r0, r11
+; CHECK-NEXT: mov r10, r3
; CHECK-NEXT: umull r2, r3, r1, r2
; CHECK-NEXT: subs r2, r1, r3
; CHECK-NEXT: add.w r2, r3, r2, lsr #1
@@ -410,18 +412,19 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: lsls r3, r3, #3
; CHECK-NEXT: sub.w r2, r3, r2, lsr #2
; CHECK-NEXT: subs r1, r2, r1
+; CHECK-NEXT: mov r3, r10
; CHECK-NEXT: add r0, r1
; CHECK-NEXT: .LBB1_3: @ %for.cond.cleanup5.loopexit134.split.loop.exit139
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: adds r7, r0, #7
+; CHECK-NEXT: add.w r11, r0, #7
; CHECK-NEXT: .LBB1_4: @ %for.cond.cleanup5
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: movs r5, #0
+; CHECK-NEXT: mov.w r10, #0
; CHECK-NEXT: .LBB1_5: @ %for.cond.cleanup5
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: add.w r11, r11, #2
-; CHECK-NEXT: subs.w r1, r11, lr
-; CHECK-NEXT: asr.w r0, r11, #31
+; CHECK-NEXT: adds r5, #2
+; CHECK-NEXT: subs.w r1, r5, lr
+; CHECK-NEXT: asr.w r0, r5, #31
; CHECK-NEXT: sbcs.w r0, r0, r12
; CHECK-NEXT: bge.w .LBB1_28
; CHECK-NEXT: .LBB1_6: @ %for.cond2.preheader
@@ -430,35 +433,36 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: @ Child Loop BB1_10 Depth 2
; CHECK-NEXT: @ Child Loop BB1_12 Depth 3
; CHECK-NEXT: @ Child Loop BB1_14 Depth 3
-; CHECK-NEXT: cmp r7, #2
+; CHECK-NEXT: cmp.w r11, #2
; CHECK-NEXT: bgt .LBB1_5
; CHECK-NEXT: @ %bb.7: @ %for.body6.lr.ph
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: ldr r0, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: cmp r0, #5
+; CHECK-NEXT: cmp r7, #5
; CHECK-NEXT: bhi .LBB1_17
; CHECK-NEXT: @ %bb.8: @ %for.body6.us.preheader
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: ldrd r2, r3, [sp, #104]
+; CHECK-NEXT: ldrd r2, r3, [sp, #120]
; CHECK-NEXT: movs r0, #32
; CHECK-NEXT: movs r1, #0
-; CHECK-NEXT: mov r6, r12
-; CHECK-NEXT: mov r4, lr
+; CHECK-NEXT: mov r4, r6
+; CHECK-NEXT: mov r7, r12
+; CHECK-NEXT: mov r6, lr
; CHECK-NEXT: bl __aeabi_ldivmod
-; CHECK-NEXT: mov lr, r4
-; CHECK-NEXT: mov r12, r6
-; CHECK-NEXT: ldr r4, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: vdup.32 q0, r2
-; CHECK-NEXT: ldr r6, [sp, #112]
-; CHECK-NEXT: mov r0, r7
+; CHECK-NEXT: mov lr, r6
+; CHECK-NEXT: mov r6, r4
+; CHECK-NEXT: mov r12, r7
; CHECK-NEXT: ldr r3, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: ldr r4, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT: vdup.32 q0, r2
+; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT: mov r0, r11
; CHECK-NEXT: b .LBB1_10
; CHECK-NEXT: .LBB1_9: @ %for.cond.cleanup17.us
; CHECK-NEXT: @ in Loop: Header=BB1_10 Depth=2
-; CHECK-NEXT: adds r7, r0, #7
+; CHECK-NEXT: add.w r11, r0, #7
; CHECK-NEXT: cmn.w r0, #4
-; CHECK-NEXT: mov.w r5, #0
-; CHECK-NEXT: mov r0, r7
+; CHECK-NEXT: mov.w r10, #0
+; CHECK-NEXT: mov r0, r11
; CHECK-NEXT: bge .LBB1_5
; CHECK-NEXT: .LBB1_10: @ %for.body6.us
; CHECK-NEXT: @ Parent Loop BB1_6 Depth=1
@@ -481,14 +485,13 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: @ Parent Loop BB1_6 Depth=1
; CHECK-NEXT: @ Parent Loop BB1_10 Depth=2
; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
-; CHECK-NEXT: vldrw.u32 q2, [r8]
-; CHECK-NEXT: subs r2, #4
-; CHECK-NEXT: vqadd.u32 q2, q2, r1
-; CHECK-NEXT: add.w r1, r1, #4
-; CHECK-NEXT: vcmp.u32 hi, q6, q2
+; CHECK-NEXT: vqadd.u32 q2, q5, r1
+; CHECK-NEXT: adds r1, #4
+; CHECK-NEXT: vcmp.u32 hi, q7, q2
; CHECK-NEXT: vshl.i32 q2, q1, #2
-; CHECK-NEXT: vadd.i32 q2, q2, r9
-; CHECK-NEXT: vadd.i32 q1, q1, r10
+; CHECK-NEXT: subs r2, #4
+; CHECK-NEXT: vadd.i32 q2, q2, r8
+; CHECK-NEXT: vadd.i32 q1, q1, r9
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q0, [q2]
; CHECK-NEXT: bne .LBB1_12
@@ -501,14 +504,13 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: @ Parent Loop BB1_6 Depth=1
; CHECK-NEXT: @ Parent Loop BB1_10 Depth=2
; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
-; CHECK-NEXT: vldrw.u32 q2, [r8]
-; CHECK-NEXT: subs r2, #4
-; CHECK-NEXT: vqadd.u32 q2, q2, r1
-; CHECK-NEXT: add.w r1, r1, #4
-; CHECK-NEXT: vcmp.u32 hi, q5, q2
+; CHECK-NEXT: vqadd.u32 q2, q5, r1
+; CHECK-NEXT: adds r1, #4
+; CHECK-NEXT: vcmp.u32 hi, q6, q2
; CHECK-NEXT: vshl.i32 q2, q1, #2
-; CHECK-NEXT: vadd.i32 q2, q2, r9
-; CHECK-NEXT: vadd.i32 q1, q1, r10
+; CHECK-NEXT: subs r2, #4
+; CHECK-NEXT: vadd.i32 q2, q2, r8
+; CHECK-NEXT: vadd.i32 q1, q1, r9
; CHECK-NEXT: vpst
; CHECK-NEXT: vstrwt.32 q0, [q2]
; CHECK-NEXT: bne .LBB1_14
@@ -518,7 +520,7 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: beq .LBB1_9
; CHECK-NEXT: @ %bb.16: @ %for.cond9.for.cond15.preheader_crit_edge.us
; CHECK-NEXT: @ in Loop: Header=BB1_10 Depth=2
-; CHECK-NEXT: eor r1, r5, #1
+; CHECK-NEXT: eor r1, r10, #1
; CHECK-NEXT: lsls r1, r1, #31
; CHECK-NEXT: bne .LBB1_9
; CHECK-NEXT: b .LBB1_26
@@ -527,11 +529,11 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: cmp r6, #0
; CHECK-NEXT: beq.w .LBB1_2
; CHECK-NEXT: @ %bb.18: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: mov r0, r7
+; CHECK-NEXT: mov r0, r11
; CHECK-NEXT: .LBB1_19: @ %for.body6.us60
; CHECK-NEXT: @ Parent Loop BB1_6 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: lsls r1, r5, #31
+; CHECK-NEXT: lsls.w r1, r10, #31
; CHECK-NEXT: bne .LBB1_27
; CHECK-NEXT: @ %bb.20: @ %for.cond.cleanup17.us63
; CHECK-NEXT: @ in Loop: Header=BB1_19 Depth=2
@@ -547,19 +549,19 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: bgt .LBB1_25
; CHECK-NEXT: @ %bb.23: @ %for.cond.cleanup17.us63.3
; CHECK-NEXT: @ in Loop: Header=BB1_19 Depth=2
-; CHECK-NEXT: add.w r7, r0, #28
+; CHECK-NEXT: add.w r11, r0, #28
; CHECK-NEXT: cmn.w r0, #25
-; CHECK-NEXT: mov.w r5, #0
-; CHECK-NEXT: mov r0, r7
+; CHECK-NEXT: mov.w r10, #0
+; CHECK-NEXT: mov r0, r11
; CHECK-NEXT: blt .LBB1_19
; CHECK-NEXT: b .LBB1_5
; CHECK-NEXT: .LBB1_24: @ %for.cond.cleanup5.loopexit134.split.loop.exit137
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: add.w r7, r0, #14
+; CHECK-NEXT: add.w r11, r0, #14
; CHECK-NEXT: b .LBB1_4
; CHECK-NEXT: .LBB1_25: @ %for.cond.cleanup5.loopexit134.split.loop.exit135
; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1
-; CHECK-NEXT: add.w r7, r0, #21
+; CHECK-NEXT: add.w r11, r0, #21
; CHECK-NEXT: b .LBB1_4
; CHECK-NEXT: .LBB1_26: @ %for.inc19.us
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
@@ -569,7 +571,7 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) {
; CHECK-NEXT: b .LBB1_27
; CHECK-NEXT: .LBB1_28: @ %for.cond.cleanup
; CHECK-NEXT: add sp, #16
-; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .p2align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
index 50f40df2d0e27..5c673dda57aef 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-increment.ll
@@ -607,71 +607,71 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_simple(ptr noalias nocapture reado
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT: .pad #32
-; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: .pad #28
+; CHECK-NEXT: sub sp, #28
; CHECK-NEXT: cmp r2, #1
-; CHECK-NEXT: strd r1, r2, [sp, #8] @ 8-byte Folded Spill
+; CHECK-NEXT: strd r1, r2, [sp, #4] @ 8-byte Folded Spill
; CHECK-NEXT: blt .LBB14_5
; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader
-; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: mov r9, r0
+; CHECK-NEXT: ldr r1, [sp, #8] @ 4-byte Reload
; CHECK-NEXT: movs r6, #1
-; CHECK-NEXT: mov.w r10, #8
+; CHECK-NEXT: add r2, sp, #12
+; CHECK-NEXT: mov.w r9, #8
; CHECK-NEXT: bic r1, r1, #7
-; CHECK-NEXT: str r1, [sp, #4] @ 4-byte Spill
-; CHECK-NEXT: sub.w r7, r1, #8
-; CHECK-NEXT: add.w r0, r6, r7, lsr #3
-; CHECK-NEXT: str r0, [sp] @ 4-byte Spill
-; CHECK-NEXT: add r0, sp, #16
+; CHECK-NEXT: str r1, [sp] @ 4-byte Spill
+; CHECK-NEXT: sub.w r3, r1, #8
+; CHECK-NEXT: add.w r8, r6, r3, lsr #3
+; CHECK-NEXT: adr r3, .LCPI14_0
+; CHECK-NEXT: vldrw.u32 q0, [r3]
; CHECK-NEXT: .LBB14_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
; CHECK-NEXT: @ Child Loop BB14_3 Depth 2
-; CHECK-NEXT: ldr r1, [sp] @ 4-byte Reload
-; CHECK-NEXT: dls lr, r1
-; CHECK-NEXT: adr r1, .LCPI14_0
-; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: dls lr, r8
+; CHECK-NEXT: vmov q1, q0
+; CHECK-NEXT: ldr r6, [sp, #4] @ 4-byte Reload
; CHECK-NEXT: .LBB14_3: @ %vector.body
; CHECK-NEXT: @ Parent Loop BB14_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vstrw.32 q0, [r0]
-; CHECK-NEXT: vadd.i16 q0, q0, r10
-; CHECK-NEXT: vldrh.s32 q1, [r0, #8]
-; CHECK-NEXT: vshl.i32 q1, q1, #1
-; CHECK-NEXT: vadd.i32 q1, q1, r9
-; CHECK-NEXT: vmov r3, r6, d3
-; CHECK-NEXT: vmov r5, r4, d2
-; CHECK-NEXT: vldrh.s32 q1, [r0]
-; CHECK-NEXT: vshl.i32 q1, q1, #1
-; CHECK-NEXT: vadd.i32 q1, q1, r9
-; CHECK-NEXT: vmov r12, r11, d3
-; CHECK-NEXT: ldrh.w r8, [r6]
-; CHECK-NEXT: vmov r2, r6, d2
+; CHECK-NEXT: vstrw.32 q1, [r2]
+; CHECK-NEXT: mov r12, r2
+; CHECK-NEXT: vldrh.s32 q2, [r2, #8]
+; CHECK-NEXT: vadd.i16 q1, q1, r9
+; CHECK-NEXT: vshl.i32 q2, q2, #1
+; CHECK-NEXT: vadd.i32 q2, q2, r0
+; CHECK-NEXT: vmov r7, r5, d5
+; CHECK-NEXT: vmov r3, r4, d4
+; CHECK-NEXT: vldrh.s32 q2, [r2]
+; CHECK-NEXT: vshl.i32 q2, q2, #1
+; CHECK-NEXT: vadd.i32 q2, q2, r0
+; CHECK-NEXT: vmov r1, r10, d5
+; CHECK-NEXT: ldrh r7, [r7]
; CHECK-NEXT: ldrh r4, [r4]
+; CHECK-NEXT: ldrh r5, [r5]
+; CHECK-NEXT: ldrh.w r2, [r10]
+; CHECK-NEXT: ldrh.w r10, [r3]
+; CHECK-NEXT: vmov r3, r11, d4
+; CHECK-NEXT: ldrh r1, [r1]
; CHECK-NEXT: ldrh r3, [r3]
-; CHECK-NEXT: ldrh.w r1, [r11]
-; CHECK-NEXT: ldrh.w r11, [r5]
-; CHECK-NEXT: ldrh.w r5, [r12]
-; CHECK-NEXT: ldrh r2, [r2]
-; CHECK-NEXT: ldrh r6, [r6]
-; CHECK-NEXT: vmov.16 q1[0], r2
-; CHECK-NEXT: vmov.16 q1[1], r6
-; CHECK-NEXT: vmov.16 q1[2], r5
-; CHECK-NEXT: vmov.16 q1[3], r1
-; CHECK-NEXT: vmov.16 q1[4], r11
-; CHECK-NEXT: vmov.16 q1[5], r4
-; CHECK-NEXT: vmov.16 q1[6], r3
-; CHECK-NEXT: vmov.16 q1[7], r8
-; CHECK-NEXT: vstrb.8 q1, [r7], #16
+; CHECK-NEXT: ldrh.w r11, [r11]
+; CHECK-NEXT: vmov.16 q2[0], r3
+; CHECK-NEXT: vmov.16 q2[1], r11
+; CHECK-NEXT: vmov.16 q2[2], r1
+; CHECK-NEXT: vmov.16 q2[3], r2
+; CHECK-NEXT: mov r2, r12
+; CHECK-NEXT: vmov.16 q2[4], r10
+; CHECK-NEXT: vmov.16 q2[5], r4
+; CHECK-NEXT: vmov.16 q2[6], r7
+; CHECK-NEXT: vmov.16 q2[7], r5
+; CHECK-NEXT: vstrb.8 q2, [r6], #16
; CHECK-NEXT: le lr, .LBB14_3
; CHECK-NEXT: @ %bb.4: @ %middle.block
; CHECK-NEXT: @ in Loop: Header=BB14_2 Depth=1
-; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: ldr r2, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: cmp r2, r1
+; CHECK-NEXT: ldr r1, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT: ldr r3, [sp] @ 4-byte Reload
+; CHECK-NEXT: cmp r3, r1
; CHECK-NEXT: bne .LBB14_2
; CHECK-NEXT: .LBB14_5: @ %for.cond.cleanup
-; CHECK-NEXT: add sp, #32
+; CHECK-NEXT: add sp, #28
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.6:
@@ -724,142 +724,143 @@ define arm_aapcs_vfpcc void @gather_inc_v8i16_complex(ptr noalias nocapture read
; CHECK-NEXT: sub sp, #4
; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT: .pad #72
-; CHECK-NEXT: sub sp, #72
+; CHECK-NEXT: .pad #120
+; CHECK-NEXT: sub sp, #120
; CHECK-NEXT: cmp r2, #1
-; CHECK-NEXT: strd r1, r2, [sp, #12] @ 8-byte Folded Spill
+; CHECK-NEXT: strd r1, r2, [sp, #64] @ 8-byte Folded Spill
; CHECK-NEXT: blt.w .LBB15_5
; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader
-; CHECK-NEXT: ldr r1, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT: add r4, sp, #56
-; CHECK-NEXT: add r7, sp, #24
-; CHECK-NEXT: add r5, sp, #40
+; CHECK-NEXT: ldr r1, [sp, #68] @ 4-byte Reload
+; CHECK-NEXT: adr r3, .LCPI15_2
+; CHECK-NEXT: vldrw.u32 q0, [r3]
+; CHECK-NEXT: movs r2, #1
; CHECK-NEXT: bic r1, r1, #7
-; CHECK-NEXT: str r1, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT: subs r1, #8
-; CHECK-NEXT: movs r3, #1
-; CHECK-NEXT: add.w r1, r3, r1, lsr #3
; CHECK-NEXT: str r1, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT: subs r1, #8
+; CHECK-NEXT: vstrw.32 q0, [sp, #40] @ 16-byte Spill
+; CHECK-NEXT: add.w r1, r2, r1, lsr #3
+; CHECK-NEXT: str r1, [sp, #60] @ 4-byte Spill
+; CHECK-NEXT: adr r1, .LCPI15_0
+; CHECK-NEXT: adr r2, .LCPI15_1
+; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: vstrw.32 q0, [sp, #24] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r2]
+; CHECK-NEXT: add r2, sp, #104
+; CHECK-NEXT: vstrw.32 q0, [sp, #8] @ 16-byte Spill
; CHECK-NEXT: .LBB15_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
; CHECK-NEXT: @ Child Loop BB15_3 Depth 2
-; CHECK-NEXT: ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT: add.w r10, sp, #88
; CHECK-NEXT: dls lr, r1
-; CHECK-NEXT: adr r1, .LCPI15_2
-; CHECK-NEXT: vldrw.u32 q1, [r1]
-; CHECK-NEXT: adr r1, .LCPI15_0
-; CHECK-NEXT: vldrw.u32 q2, [r1]
-; CHECK-NEXT: adr r1, .LCPI15_1
-; CHECK-NEXT: ldr.w r12, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: vldrw.u32 q3, [r1]
+; CHECK-NEXT: ldr r7, [sp, #64] @ 4-byte Reload
+; CHECK-NEXT: vldrw.u32 q4, [sp, #24] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q5, [sp, #40] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q6, [sp, #8] @ 16-byte Reload
; CHECK-NEXT: .LBB15_3: @ %vector.body
; CHECK-NEXT: @ Parent Loop BB15_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vstrw.32 q1, [r4]
-; CHECK-NEXT: mov r1, r5
-; CHECK-NEXT: vldrh.s32 q0, [r4, #8]
-; CHECK-NEXT: mov r11, r4
-; CHECK-NEXT: mov r5, r7
+; CHECK-NEXT: vstrw.32 q5, [r2]
+; CHECK-NEXT: mov r8, r2
+; CHECK-NEXT: vldrh.s32 q0, [r2, #8]
; CHECK-NEXT: vshl.i32 q0, q0, #1
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov r2, r3, d0
-; CHECK-NEXT: vmov r6, r10, d1
-; CHECK-NEXT: vldrh.s32 q0, [r4]
+; CHECK-NEXT: vmov r1, r3, d0
+; CHECK-NEXT: vmov r4, r5, d1
+; CHECK-NEXT: vldrh.s32 q0, [r2]
; CHECK-NEXT: vshl.i32 q0, q0, #1
-; CHECK-NEXT: vadd.i32 q6, q0, r0
-; CHECK-NEXT: vmov r7, r4, d12
-; CHECK-NEXT: ldrh.w r9, [r2]
-; CHECK-NEXT: ldrh.w r2, [r10]
-; CHECK-NEXT: str r2, [sp, #20] @ 4-byte Spill
-; CHECK-NEXT: ldrh.w r8, [r3]
-; CHECK-NEXT: ldrh r3, [r6]
-; CHECK-NEXT: ldrh r2, [r7]
-; CHECK-NEXT: mov r7, r5
-; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: vstrw.32 q3, [r7]
-; CHECK-NEXT: vldrh.s32 q0, [r7]
-; CHECK-NEXT: vmov.16 q4[0], r2
-; CHECK-NEXT: vmov.16 q4[1], r4
-; CHECK-NEXT: mov r5, r1
+; CHECK-NEXT: vadd.i32 q2, q0, r0
+; CHECK-NEXT: vmov r6, r2, d4
+; CHECK-NEXT: ldrh r1, [r1]
+; CHECK-NEXT: ldrh.w r12, [r4]
+; CHECK-NEXT: add r4, sp, #72
+; CHECK-NEXT: ldrh.w r11, [r5]
+; CHECK-NEXT: ldrh r3, [r3]
+; CHECK-NEXT: ldrh r5, [r6]
+; CHECK-NEXT: ldrh r2, [r2]
+; CHECK-NEXT: vstrw.32 q6, [r4]
+; CHECK-NEXT: vldrh.s32 q0, [r4]
+; CHECK-NEXT: vmov.16 q7[0], r5
+; CHECK-NEXT: vmov.16 q7[1], r2
; CHECK-NEXT: vshl.i32 q0, q0, #1
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov r4, r6, d0
-; CHECK-NEXT: vmov r1, r2, d1
-; CHECK-NEXT: vldrh.s32 q0, [r7, #8]
+; CHECK-NEXT: vmov r6, r9, d0
+; CHECK-NEXT: vmov r2, r5, d1
+; CHECK-NEXT: vldrh.s32 q0, [r4, #8]
; CHECK-NEXT: vshl.i32 q0, q0, #1
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: ldrh r1, [r1]
-; CHECK-NEXT: vmov.16 q5[0], r4
-; CHECK-NEXT: ldrh r4, [r6]
+; CHECK-NEXT: ldrh r6, [r6]
; CHECK-NEXT: ldrh r2, [r2]
-; CHECK-NEXT: vmov.16 q5[1], r4
-; CHECK-NEXT: vmov.16 q5[2], r1
-; CHECK-NEXT: vmov r1, r4, d0
-; CHECK-NEXT: vmov.16 q5[3], r2
-; CHECK-NEXT: ldrh r1, [r1]
-; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: vmov.16 q5[4], r1
-; CHECK-NEXT: vmov r1, r2, d1
-; CHECK-NEXT: vmov.16 q5[5], r4
-; CHECK-NEXT: ldrh r1, [r1]
+; CHECK-NEXT: vmov.16 q1[0], r6
+; CHECK-NEXT: ldrh.w r6, [r9]
+; CHECK-NEXT: ldrh r5, [r5]
+; CHECK-NEXT: vmov.16 q1[1], r6
+; CHECK-NEXT: vmov.16 q1[2], r2
+; CHECK-NEXT: vmov r2, r6, d0
+; CHECK-NEXT: vmov.16 q1[3], r5
; CHECK-NEXT: ldrh r2, [r2]
-; CHECK-NEXT: vstrw.32 q2, [r5]
-; CHECK-NEXT: vldrh.s32 q0, [r5]
-; CHECK-NEXT: vmov.16 q5[6], r1
-; CHECK-NEXT: vmov.16 q5[7], r2
+; CHECK-NEXT: ldrh r6, [r6]
+; CHECK-NEXT: vmov.16 q1[4], r2
+; CHECK-NEXT: vmov r2, r5, d1
+; CHECK-NEXT: vmov.16 q1[5], r6
+; CHECK-NEXT: mov r6, r10
+; CHECK-NEXT: ldrh r2, [r2]
+; CHECK-NEXT: ldrh r5, [r5]
+; CHECK-NEXT: vstrw.32 q4, [r10]
+; CHECK-NEXT: vldrh.s32 q0, [r6]
+; CHECK-NEXT: vmov.16 q1[6], r2
+; CHECK-NEXT: vmov.16 q1[7], r5
; CHECK-NEXT: vshl.i32 q0, q0, #1
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov r1, r2, d0
-; CHECK-NEXT: ldrh r1, [r1]
+; CHECK-NEXT: vmov r2, r5, d0
; CHECK-NEXT: ldrh r2, [r2]
-; CHECK-NEXT: vmov.16 q7[0], r1
-; CHECK-NEXT: vmov.16 q7[1], r2
-; CHECK-NEXT: vmov r1, r2, d13
-; CHECK-NEXT: vmov.i16 q6, #0x18
-; CHECK-NEXT: vadd.i16 q3, q3, q6
-; CHECK-NEXT: vadd.i16 q1, q1, q6
-; CHECK-NEXT: vadd.i16 q2, q2, q6
-; CHECK-NEXT: ldrh.w r10, [r2]
+; CHECK-NEXT: ldrh r5, [r5]
+; CHECK-NEXT: vmov.16 q3[0], r2
+; CHECK-NEXT: vmov.16 q3[1], r5
+; CHECK-NEXT: vmov r2, r5, d5
+; CHECK-NEXT: vmov.i16 q2, #0x18
+; CHECK-NEXT: vadd.i16 q6, q6, q2
+; CHECK-NEXT: vadd.i16 q5, q5, q2
+; CHECK-NEXT: vadd.i16 q4, q4, q2
+; CHECK-NEXT: ldrh.w r9, [r2]
; CHECK-NEXT: vmov r2, r4, d1
-; CHECK-NEXT: vldrh.s32 q0, [r5, #8]
-; CHECK-NEXT: ldrh r1, [r1]
+; CHECK-NEXT: vldrh.s32 q0, [r6, #8]
+; CHECK-NEXT: ldrh r5, [r5]
+; CHECK-NEXT: vmov.16 q7[2], r9
; CHECK-NEXT: vshl.i32 q0, q0, #1
-; CHECK-NEXT: vmov.16 q4[2], r1
+; CHECK-NEXT: vmov.16 q7[3], r5
; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov.16 q4[3], r10
-; CHECK-NEXT: vmov.16 q4[4], r9
-; CHECK-NEXT: ldr r1, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT: vmov.16 q4[5], r8
-; CHECK-NEXT: vmov.16 q4[6], r3
-; CHECK-NEXT: vmov.16 q4[7], r1
+; CHECK-NEXT: vmov.16 q7[4], r1
+; CHECK-NEXT: vmov.16 q7[5], r3
+; CHECK-NEXT: vmov.16 q7[6], r12
+; CHECK-NEXT: vmov.16 q7[7], r11
; CHECK-NEXT: ldrh r2, [r2]
; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: vmov.16 q7[2], r2
-; CHECK-NEXT: vmov.16 q7[3], r4
+; CHECK-NEXT: vmov.16 q3[2], r2
+; CHECK-NEXT: vmov.16 q3[3], r4
; CHECK-NEXT: vmov r2, r4, d0
; CHECK-NEXT: ldrh r2, [r2]
; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: vmov.16 q7[4], r2
-; CHECK-NEXT: vmov.16 q7[5], r4
+; CHECK-NEXT: vmov.16 q3[4], r2
+; CHECK-NEXT: vmov.16 q3[5], r4
; CHECK-NEXT: vmov r2, r4, d1
; CHECK-NEXT: ldrh r2, [r2]
; CHECK-NEXT: ldrh r4, [r4]
-; CHECK-NEXT: vmov.16 q7[6], r2
-; CHECK-NEXT: vmov.16 q7[7], r4
-; CHECK-NEXT: mov r4, r11
-; CHECK-NEXT: vadd.i16 q0, q7, q5
-; CHECK-NEXT: vadd.i16 q0, q0, q4
-; CHECK-NEXT: vstrb.8 q0, [r12], #16
+; CHECK-NEXT: vmov.16 q3[6], r2
+; CHECK-NEXT: mov r2, r8
+; CHECK-NEXT: vmov.16 q3[7], r4
+; CHECK-NEXT: vadd.i16 q0, q3, q1
+; CHECK-NEXT: vadd.i16 q0, q0, q7
+; CHECK-NEXT: vstrb.8 q0, [r7], #16
; CHECK-NEXT: le lr, .LBB15_3
; CHECK-NEXT: @ %bb.4: @ %middle.block
; CHECK-NEXT: @ in Loop: Header=BB15_2 Depth=1
-; CHECK-NEXT: ldr r1, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: ldr r2, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT: cmp r1, r2
+; CHECK-NEXT: ldr r1, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: ldr r3, [sp, #68] @ 4-byte Reload
+; CHECK-NEXT: cmp r1, r3
; CHECK-NEXT: bne.w .LBB15_2
; CHECK-NEXT: .LBB15_5: @ %for.cond.cleanup
-; CHECK-NEXT: add sp, #72
+; CHECK-NEXT: add sp, #120
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
@@ -935,243 +936,260 @@ for.cond.cleanup: ; preds = %for.body, %middle.b
define arm_aapcs_vfpcc void @gather_inc_v16i8_complex(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n) {
; CHECK-LABEL: gather_inc_v16i8_complex:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: cmp r2, #1
-; CHECK-NEXT: it lt
-; CHECK-NEXT: bxlt lr
-; CHECK-NEXT: .LBB16_1: @ %vector.ph.preheader
; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: .pad #4
; CHECK-NEXT: sub sp, #4
; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-NEXT: .pad #160
-; CHECK-NEXT: sub sp, #160
-; CHECK-NEXT: bic lr, r2, #7
-; CHECK-NEXT: mov r12, r1
-; CHECK-NEXT: .LBB16_2: @ %vector.ph
-; CHECK-NEXT: @ =>This Loop Header: Depth=1
-; CHECK-NEXT: @ Child Loop BB16_3 Depth 2
+; CHECK-NEXT: .pad #312
+; CHECK-NEXT: sub sp, #312
+; CHECK-NEXT: cmp r2, #1
+; CHECK-NEXT: str r1, [sp, #116] @ 4-byte Spill
+; CHECK-NEXT: blt.w .LBB16_5
+; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader
; CHECK-NEXT: adr r1, .LCPI16_0
-; CHECK-NEXT: mov r8, r12
-; CHECK-NEXT: vldrw.u32 q1, [r1]
-; CHECK-NEXT: adr r1, .LCPI16_1
+; CHECK-NEXT: adr r6, .LCPI16_8
; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: adr r1, .LCPI16_8
-; CHECK-NEXT: mov r9, lr
-; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT: adr r1, .LCPI16_1
+; CHECK-NEXT: adr r7, .LCPI16_7
+; CHECK-NEXT: adr r3, .LCPI16_6
+; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: adr r1, .LCPI16_7
-; CHECK-NEXT: vldrw.u32 q3, [r1]
-; CHECK-NEXT: adr r1, .LCPI16_9
-; CHECK-NEXT: vldrw.u32 q4, [r1]
; CHECK-NEXT: adr r1, .LCPI16_5
+; CHECK-NEXT: bic r10, r2, #7
; CHECK-NEXT: vstrw.32 q0, [sp, #80] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r6]
+; CHECK-NEXT: adr r6, .LCPI16_9
+; CHECK-NEXT: vstrw.32 q0, [sp, #64] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r7]
+; CHECK-NEXT: vstrw.32 q0, [sp, #48] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r6]
+; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: adr r1, .LCPI16_6
-; CHECK-NEXT: vldrw.u32 q7, [r1]
+; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r3]
+; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
+; CHECK-NEXT: .LBB16_2: @ %vector.ph
+; CHECK-NEXT: @ =>This Loop Header: Depth=1
+; CHECK-NEXT: @ Child Loop BB16_3 Depth 2
+; CHECK-NEXT: vldrw.u32 q2, [sp, #16] @ 16-byte Reload
; CHECK-NEXT: adr r1, .LCPI16_3
; CHECK-NEXT: vldrw.u32 q5, [r1]
; CHECK-NEXT: adr r1, .LCPI16_4
-; CHECK-NEXT: vstrw.32 q0, [sp, #144] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: vstrw.32 q2, [sp, #296] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q2, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q6, [r1]
; CHECK-NEXT: adr r1, .LCPI16_2
-; CHECK-NEXT: vstrw.32 q0, [sp, #128] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: vstrw.32 q2, [sp, #280] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q2, [sp, #48] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q1, [r1]
; CHECK-NEXT: adr r1, .LCPI16_10
-; CHECK-NEXT: vstrw.32 q0, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q0, [r1]
+; CHECK-NEXT: vstrw.32 q2, [sp, #264] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q2, [sp, #80] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q3, [r1]
; CHECK-NEXT: adr r1, .LCPI16_11
-; CHECK-NEXT: vldrw.u32 q2, [r1]
-; CHECK-NEXT: vstrw.32 q0, [sp, #96] @ 16-byte Spill
+; CHECK-NEXT: ldr.w r8, [sp, #116] @ 4-byte Reload
+; CHECK-NEXT: vstrw.32 q2, [sp, #248] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q2, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q7, [r1]
+; CHECK-NEXT: vldrw.u32 q4, [sp] @ 16-byte Reload
+; CHECK-NEXT: mov r11, r10
+; CHECK-NEXT: vstrw.32 q2, [sp, #232] @ 16-byte Spill
+; CHECK-NEXT: vstrw.32 q0, [sp, #216] @ 16-byte Spill
; CHECK-NEXT: .LBB16_3: @ %vector.body
; CHECK-NEXT: @ Parent Loop BB16_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vadd.i32 q6, q5, r0
-; CHECK-NEXT: vstrw.32 q7, [sp, #16] @ 16-byte Spill
-; CHECK-NEXT: vadd.i32 q7, q7, r0
-; CHECK-NEXT: vmov r10, r1, d13
-; CHECK-NEXT: vmov r7, r11, d14
-; CHECK-NEXT: vstrw.32 q5, [sp, #32] @ 16-byte Spill
-; CHECK-NEXT: vadd.i32 q5, q2, r0
-; CHECK-NEXT: vldrw.u32 q0, [sp, #112] @ 16-byte Reload
-; CHECK-NEXT: vmov r5, r3, d11
-; CHECK-NEXT: vstrw.32 q4, [sp, #48] @ 16-byte Spill
-; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vldrw.u32 q4, [sp, #48] @ 16-byte Reload
-; CHECK-NEXT: vstrw.32 q1, [sp] @ 16-byte Spill
-; CHECK-NEXT: subs.w r9, r9, #16
-; CHECK-NEXT: vldrw.u32 q1, [sp] @ 16-byte Reload
-; CHECK-NEXT: ldrb r6, [r1]
-; CHECK-NEXT: ldrb r1, [r7]
-; CHECK-NEXT: vmov r7, r4, d10
+; CHECK-NEXT: vmov q0, q7
+; CHECK-NEXT: vstrw.32 q7, [sp, #184] @ 16-byte Spill
+; CHECK-NEXT: vadd.i32 q7, q5, r0
+; CHECK-NEXT: vstrw.32 q5, [sp, #200] @ 16-byte Spill
+; CHECK-NEXT: vadd.i32 q5, q0, r0
+; CHECK-NEXT: vmov q0, q6
+; CHECK-NEXT: vadd.i32 q6, q4, r0
+; CHECK-NEXT: vmov r5, r4, d11
+; CHECK-NEXT: vmov r1, lr, d12
+; CHECK-NEXT: vadd.i32 q2, q1, r0
+; CHECK-NEXT: vmov r6, r7, d15
+; CHECK-NEXT: vstrw.32 q1, [sp, #152] @ 16-byte Spill
+; CHECK-NEXT: vmov q1, q3
+; CHECK-NEXT: vstrw.32 q4, [sp, #168] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q4, [sp, #248] @ 16-byte Reload
+; CHECK-NEXT: vstrw.32 q0, [sp, #120] @ 16-byte Spill
+; CHECK-NEXT: vstrw.32 q3, [sp, #136] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q3, [sp, #184] @ 16-byte Reload
+; CHECK-NEXT: subs.w r11, r11, #16
; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: ldrb r3, [r3]
+; CHECK-NEXT: ldrb.w r9, [r1]
+; CHECK-NEXT: vmov r1, r3, d10
; CHECK-NEXT: ldrb r7, [r7]
-; CHECK-NEXT: ldrb r4, [r4]
-; CHECK-NEXT: vmov.8 q5[0], r7
-; CHECK-NEXT: vmov.8 q5[1], r4
+; CHECK-NEXT: ldrb r1, [r1]
+; CHECK-NEXT: vmov.8 q5[0], r1
+; CHECK-NEXT: ldrb r1, [r3]
+; CHECK-NEXT: vmov.8 q5[1], r1
+; CHECK-NEXT: vmov r1, r3, d14
; CHECK-NEXT: vmov.8 q5[2], r5
-; CHECK-NEXT: vmov r4, r5, d12
-; CHECK-NEXT: vmov.8 q5[3], r3
-; CHECK-NEXT: ldrb r4, [r4]
-; CHECK-NEXT: ldrb r7, [r5]
-; CHECK-NEXT: vmov.8 q6[0], r4
-; CHECK-NEXT: ldrb.w r5, [r10]
-; CHECK-NEXT: vmov.8 q6[1], r7
-; CHECK-NEXT: vmov r3, r4, d0
-; CHECK-NEXT: vmov.8 q6[2], r5
-; CHECK-NEXT: vmov r5, r10, d1
-; CHECK-NEXT: vmov.8 q6[3], r6
-; CHECK-NEXT: vldrw.u32 q0, [sp, #96] @ 16-byte Reload
-; CHECK-NEXT: vmov.8 q6[4], r1
-; CHECK-NEXT: ldrb.w r7, [r11]
-; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov.8 q6[5], r7
-; CHECK-NEXT: ldrb r4, [r4]
-; CHECK-NEXT: ldrb r6, [r5]
-; CHECK-NEXT: vmov r1, r5, d15
-; CHECK-NEXT: vldrw.u32 q7, [sp, #64] @ 16-byte Reload
+; CHECK-NEXT: ldrb r5, [r6]
+; CHECK-NEXT: ldrb r6, [r4]
+; CHECK-NEXT: vmov.8 q5[3], r6
; CHECK-NEXT: ldrb r1, [r1]
+; CHECK-NEXT: ldrb r3, [r3]
+; CHECK-NEXT: vmov.8 q7[0], r1
+; CHECK-NEXT: vmov r6, r1, d4
+; CHECK-NEXT: vmov.8 q7[1], r3
+; CHECK-NEXT: vmov.8 q7[2], r5
+; CHECK-NEXT: vmov.8 q7[3], r7
+; CHECK-NEXT: ldrb.w r7, [lr]
+; CHECK-NEXT: vmov.8 q7[4], r9
+; CHECK-NEXT: vmov.8 q7[5], r7
+; CHECK-NEXT: ldrb r4, [r1]
+; CHECK-NEXT: vmov r1, r5, d5
+; CHECK-NEXT: vadd.i32 q2, q1, r0
+; CHECK-NEXT: vldrw.u32 q1, [sp, #280] @ 16-byte Reload
+; CHECK-NEXT: ldrb.w r12, [r1]
+; CHECK-NEXT: vmov r1, r3, d13
; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: vmov.8 q6[6], r1
-; CHECK-NEXT: vmov r1, r7, d0
-; CHECK-NEXT: vmov.8 q6[7], r5
+; CHECK-NEXT: vldrw.u32 q6, [sp, #232] @ 16-byte Reload
+; CHECK-NEXT: ldrb r1, [r1]
+; CHECK-NEXT: ldrb r3, [r3]
+; CHECK-NEXT: vmov.8 q7[6], r1
+; CHECK-NEXT: vmov r1, r7, d4
+; CHECK-NEXT: vmov.8 q7[7], r3
; CHECK-NEXT: ldrb r1, [r1]
; CHECK-NEXT: ldrb r7, [r7]
; CHECK-NEXT: vmov.8 q5[4], r1
-; CHECK-NEXT: vmov r1, r5, d1
+; CHECK-NEXT: vmov r1, r3, d5
; CHECK-NEXT: vmov.8 q5[5], r7
-; CHECK-NEXT: vadd.i32 q0, q4, r0
+; CHECK-NEXT: vadd.i32 q2, q1, r0
+; CHECK-NEXT: vldrw.u32 q1, [sp, #296] @ 16-byte Reload
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: ldrb r5, [r5]
+; CHECK-NEXT: ldrb r3, [r3]
; CHECK-NEXT: vmov.8 q5[6], r1
-; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q5[7], r5
-; CHECK-NEXT: vmov r3, r7, d0
+; CHECK-NEXT: ldrb r1, [r6]
+; CHECK-NEXT: vmov.8 q5[7], r3
+; CHECK-NEXT: vmov r7, r6, d4
+; CHECK-NEXT: vmov r3, lr, d5
; CHECK-NEXT: vmov.8 q5[8], r1
-; CHECK-NEXT: vmov r1, r11, d1
-; CHECK-NEXT: vldrw.u32 q0, [sp, #144] @ 16-byte Reload
+; CHECK-NEXT: vadd.i32 q2, q1, r0
; CHECK-NEXT: vmov.8 q5[9], r4
-; CHECK-NEXT: vmov.8 q5[10], r6
-; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: vmov r5, r6, d0
-; CHECK-NEXT: ldrb r4, [r7]
-; CHECK-NEXT: ldrb.w r7, [r10]
+; CHECK-NEXT: vmov r4, r1, d4
+; CHECK-NEXT: vmov.8 q5[10], r12
+; CHECK-NEXT: vmov.8 q5[11], r5
+; CHECK-NEXT: vldrw.u32 q1, [sp, #264] @ 16-byte Reload
+; CHECK-NEXT: ldrb r7, [r7]
+; CHECK-NEXT: ldrb r6, [r6]
; CHECK-NEXT: ldrb r3, [r3]
-; CHECK-NEXT: vmov.8 q5[11], r7
+; CHECK-NEXT: ldrb r4, [r4]
; CHECK-NEXT: ldrb r1, [r1]
+; CHECK-NEXT: vmov.8 q7[8], r4
+; CHECK-NEXT: vmov r5, r4, d5
+; CHECK-NEXT: vmov.8 q7[9], r1
+; CHECK-NEXT: vadd.i32 q2, q0, r0
+; CHECK-NEXT: vldrw.u32 q0, [sp, #216] @ 16-byte Reload
; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: ldrb r6, [r6]
-; CHECK-NEXT: vmov.8 q6[8], r5
-; CHECK-NEXT: vmov r5, r7, d1
-; CHECK-NEXT: vmov.8 q6[9], r6
-; CHECK-NEXT: vldrw.u32 q0, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT: vadd.i32 q0, q0, r0
-; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: ldrb r7, [r7]
-; CHECK-NEXT: vmov.8 q6[10], r5
-; CHECK-NEXT: vmov.8 q6[11], r7
-; CHECK-NEXT: vmov.8 q6[12], r3
-; CHECK-NEXT: vmov.8 q6[13], r4
-; CHECK-NEXT: vmov.8 q6[14], r1
-; CHECK-NEXT: vmov r1, r3, d0
+; CHECK-NEXT: ldrb r4, [r4]
+; CHECK-NEXT: vmov.8 q7[10], r5
+; CHECK-NEXT: vmov.8 q7[11], r4
+; CHECK-NEXT: vmov.8 q7[12], r7
+; CHECK-NEXT: vmov.8 q7[13], r6
+; CHECK-NEXT: vmov.8 q7[14], r3
+; CHECK-NEXT: vmov r1, r3, d4
; CHECK-NEXT: ldrb r1, [r1]
; CHECK-NEXT: vmov.8 q5[12], r1
; CHECK-NEXT: ldrb r1, [r3]
; CHECK-NEXT: vmov.8 q5[13], r1
-; CHECK-NEXT: vmov r1, r3, d1
-; CHECK-NEXT: vadd.i32 q0, q3, r0
+; CHECK-NEXT: vmov r1, r3, d5
+; CHECK-NEXT: vadd.i32 q2, q1, r0
; CHECK-NEXT: ldrb r1, [r1]
; CHECK-NEXT: vmov.8 q5[14], r1
; CHECK-NEXT: ldrb r1, [r3]
; CHECK-NEXT: vmov.8 q5[15], r1
-; CHECK-NEXT: ldrb.w r1, [r11]
-; CHECK-NEXT: vmov.8 q6[15], r1
-; CHECK-NEXT: vmov r1, r3, d0
-; CHECK-NEXT: vadd.i8 q5, q6, q5
+; CHECK-NEXT: ldrb.w r1, [lr]
+; CHECK-NEXT: vmov.8 q7[15], r1
+; CHECK-NEXT: vmov r1, r3, d4
+; CHECK-NEXT: vadd.i8 q5, q7, q5
; CHECK-NEXT: ldrb r1, [r1]
; CHECK-NEXT: ldrb r3, [r3]
-; CHECK-NEXT: vmov.8 q6[0], r1
-; CHECK-NEXT: vmov.8 q6[1], r3
-; CHECK-NEXT: vmov r1, r3, d1
-; CHECK-NEXT: vadd.i32 q0, q7, r0
+; CHECK-NEXT: vmov.8 q7[0], r1
+; CHECK-NEXT: vmov.8 q7[1], r3
+; CHECK-NEXT: vmov r1, r3, d5
+; CHECK-NEXT: vadd.i32 q2, q4, r0
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[2], r1
+; CHECK-NEXT: vmov.8 q7[2], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[3], r1
-; CHECK-NEXT: vmov r1, r3, d0
+; CHECK-NEXT: vmov.8 q7[3], r1
+; CHECK-NEXT: vmov r1, r3, d4
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[4], r1
+; CHECK-NEXT: vmov.8 q7[4], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[5], r1
-; CHECK-NEXT: vmov r1, r3, d1
-; CHECK-NEXT: vldrw.u32 q0, [sp, #80] @ 16-byte Reload
-; CHECK-NEXT: vadd.i32 q0, q0, r0
+; CHECK-NEXT: vmov.8 q7[5], r1
+; CHECK-NEXT: vmov r1, r3, d5
+; CHECK-NEXT: vadd.i32 q2, q6, r0
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[6], r1
+; CHECK-NEXT: vmov.8 q7[6], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[7], r1
-; CHECK-NEXT: vmov r1, r3, d0
+; CHECK-NEXT: vmov.8 q7[7], r1
+; CHECK-NEXT: vmov r1, r3, d4
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[8], r1
+; CHECK-NEXT: vmov.8 q7[8], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[9], r1
-; CHECK-NEXT: vmov r1, r3, d1
-; CHECK-NEXT: vadd.i32 q0, q1, r0
+; CHECK-NEXT: vmov.8 q7[9], r1
+; CHECK-NEXT: vmov r1, r3, d5
+; CHECK-NEXT: vadd.i32 q2, q0, r0
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[10], r1
+; CHECK-NEXT: vmov.8 q7[10], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[11], r1
-; CHECK-NEXT: vmov r1, r3, d0
+; CHECK-NEXT: vmov.8 q7[11], r1
+; CHECK-NEXT: vmov r1, r3, d4
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[12], r1
+; CHECK-NEXT: vmov.8 q7[12], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[13], r1
-; CHECK-NEXT: vmov r1, r3, d1
+; CHECK-NEXT: vmov.8 q7[13], r1
+; CHECK-NEXT: vmov r1, r3, d5
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q6[14], r1
+; CHECK-NEXT: vmov.8 q7[14], r1
; CHECK-NEXT: ldrb r1, [r3]
-; CHECK-NEXT: vmov.8 q6[15], r1
-; CHECK-NEXT: vadd.i8 q0, q5, q6
-; CHECK-NEXT: vldrw.u32 q6, [sp, #96] @ 16-byte Reload
-; CHECK-NEXT: vstrb.8 q0, [r8], #16
-; CHECK-NEXT: vmov.i32 q0, #0x30
-; CHECK-NEXT: vadd.i32 q6, q6, q0
-; CHECK-NEXT: vadd.i32 q7, q7, q0
-; CHECK-NEXT: vstrw.32 q6, [sp, #96] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q6, [sp, #112] @ 16-byte Reload
-; CHECK-NEXT: vldrw.u32 q5, [sp, #32] @ 16-byte Reload
-; CHECK-NEXT: vstrw.32 q7, [sp, #64] @ 16-byte Spill
-; CHECK-NEXT: vadd.i32 q6, q6, q0
-; CHECK-NEXT: vldrw.u32 q7, [sp, #16] @ 16-byte Reload
-; CHECK-NEXT: vstrw.32 q6, [sp, #112] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q6, [sp, #80] @ 16-byte Reload
-; CHECK-NEXT: vadd.i32 q2, q2, q0
-; CHECK-NEXT: vadd.i32 q3, q3, q0
-; CHECK-NEXT: vadd.i32 q6, q6, q0
-; CHECK-NEXT: vadd.i32 q5, q5, q0
-; CHECK-NEXT: vstrw.32 q6, [sp, #80] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q6, [sp, #144] @ 16-byte Reload
-; CHECK-NEXT: vadd.i32 q7, q7, q0
-; CHECK-NEXT: vadd.i32 q1, q1, q0
-; CHECK-NEXT: vadd.i32 q6, q6, q0
-; CHECK-NEXT: vadd.i32 q4, q4, q0
-; CHECK-NEXT: vstrw.32 q6, [sp, #144] @ 16-byte Spill
-; CHECK-NEXT: vldrw.u32 q6, [sp, #128] @ 16-byte Reload
-; CHECK-NEXT: vadd.i32 q6, q6, q0
-; CHECK-NEXT: vstrw.32 q6, [sp, #128] @ 16-byte Spill
+; CHECK-NEXT: vmov.8 q7[15], r1
+; CHECK-NEXT: vadd.i8 q2, q5, q7
+; CHECK-NEXT: vldrw.u32 q5, [sp, #200] @ 16-byte Reload
+; CHECK-NEXT: vstrb.8 q2, [r8], #16
+; CHECK-NEXT: vmov.i32 q2, #0x30
+; CHECK-NEXT: vadd.i32 q6, q6, q2
+; CHECK-NEXT: vadd.i32 q3, q3, q2
+; CHECK-NEXT: vstrw.32 q6, [sp, #232] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q6, [sp, #296] @ 16-byte Reload
+; CHECK-NEXT: vadd.i32 q1, q1, q2
+; CHECK-NEXT: vadd.i32 q4, q4, q2
+; CHECK-NEXT: vadd.i32 q6, q6, q2
+; CHECK-NEXT: vadd.i32 q0, q0, q2
+; CHECK-NEXT: vmov q7, q3
+; CHECK-NEXT: vldrw.u32 q3, [sp, #136] @ 16-byte Reload
+; CHECK-NEXT: vstrw.32 q1, [sp, #264] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q1, [sp, #152] @ 16-byte Reload
+; CHECK-NEXT: vstrw.32 q4, [sp, #248] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q4, [sp, #168] @ 16-byte Reload
+; CHECK-NEXT: vstrw.32 q6, [sp, #296] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q6, [sp, #120] @ 16-byte Reload
+; CHECK-NEXT: vstrw.32 q0, [sp, #216] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [sp, #280] @ 16-byte Reload
+; CHECK-NEXT: vadd.i32 q5, q5, q2
+; CHECK-NEXT: vadd.i32 q3, q3, q2
+; CHECK-NEXT: vadd.i32 q0, q0, q2
+; CHECK-NEXT: vadd.i32 q4, q4, q2
+; CHECK-NEXT: vadd.i32 q1, q1, q2
+; CHECK-NEXT: vadd.i32 q6, q6, q2
+; CHECK-NEXT: vstrw.32 q0, [sp, #280] @ 16-byte Spill
; CHECK-NEXT: bne.w .LBB16_3
; CHECK-NEXT: @ %bb.4: @ %middle.block
; CHECK-NEXT: @ in Loop: Header=BB16_2 Depth=1
-; CHECK-NEXT: cmp lr, r2
+; CHECK-NEXT: cmp r10, r2
; CHECK-NEXT: bne.w .LBB16_2
-; CHECK-NEXT: @ %bb.5:
-; CHECK-NEXT: add sp, #160
+; CHECK-NEXT: .LBB16_5: @ %for.cond.cleanup
+; CHECK-NEXT: add sp, #312
; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
-; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
-; CHECK-NEXT: bx lr
+; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.6:
; CHECK-NEXT: .LCPI16_0:
@@ -1280,95 +1298,102 @@ define arm_aapcs_vfpcc void @gather_inc_v16i8_simple(ptr noalias nocapture reado
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: .pad #4
; CHECK-NEXT: sub sp, #4
-; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: .pad #16
-; CHECK-NEXT: sub sp, #16
+; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT: .pad #64
+; CHECK-NEXT: sub sp, #64
; CHECK-NEXT: cmp r2, #1
-; CHECK-NEXT: strd r1, r2, [sp, #8] @ 8-byte Folded Spill
-; CHECK-NEXT: blt .LBB17_5
+; CHECK-NEXT: strd r1, r2, [sp, #56] @ 8-byte Folded Spill
+; CHECK-NEXT: blt.w .LBB17_5
; CHECK-NEXT: @ %bb.1: @ %vector.ph.preheader
-; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: mov.w r11, #16
-; CHECK-NEXT: bic r3, r1, #7
-; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT: adr r5, .LCPI17_3
+; CHECK-NEXT: adr r7, .LCPI17_1
+; CHECK-NEXT: vldrw.u32 q0, [r5]
+; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT: adr r3, .LCPI17_0
+; CHECK-NEXT: adr r6, .LCPI17_2
+; CHECK-NEXT: vstrw.32 q0, [sp, #32] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r7]
+; CHECK-NEXT: bic r9, r1, #7
+; CHECK-NEXT: vldrw.u32 q3, [r3]
+; CHECK-NEXT: vstrw.32 q0, [sp, #16] @ 16-byte Spill
+; CHECK-NEXT: vldrw.u32 q0, [r6]
+; CHECK-NEXT: mov.w lr, #16
+; CHECK-NEXT: str.w r9, [sp, #52] @ 4-byte Spill
+; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill
; CHECK-NEXT: .LBB17_2: @ %vector.ph
; CHECK-NEXT: @ =>This Loop Header: Depth=1
; CHECK-NEXT: @ Child Loop BB17_3 Depth 2
-; CHECK-NEXT: adr r1, .LCPI17_3
-; CHECK-NEXT: ldr r7, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: vldrw.u32 q5, [r1]
-; CHECK-NEXT: adr r1, .LCPI17_1
-; CHECK-NEXT: vldrw.u32 q4, [r1]
-; CHECK-NEXT: adr r1, .LCPI17_2
-; CHECK-NEXT: vldrw.u32 q0, [r1]
-; CHECK-NEXT: adr r1, .LCPI17_0
-; CHECK-NEXT: vldrw.u32 q1, [r1]
+; CHECK-NEXT: ldr.w r8, [sp, #56] @ 4-byte Reload
+; CHECK-NEXT: vldrw.u32 q5, [sp] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload
+; CHECK-NEXT: vldrw.u32 q7, [sp, #32] @ 16-byte Reload
+; CHECK-NEXT: vmov q4, q3
; CHECK-NEXT: .LBB17_3: @ %vector.body
; CHECK-NEXT: @ Parent Loop BB17_2 Depth=1
; CHECK-NEXT: @ => This Inner Loop Header: Depth=2
-; CHECK-NEXT: vadd.i32 q6, q1, r0
-; CHECK-NEXT: vadd.i32 q2, q0, r0
-; CHECK-NEXT: vmov r4, r5, d13
-; CHECK-NEXT: vadd.i32 q3, q5, r11
-; CHECK-NEXT: vmov lr, r8, d4
-; CHECK-NEXT: subs r3, #16
-; CHECK-NEXT: vmov r6, r12, d5
-; CHECK-NEXT: vadd.i32 q2, q4, r11
-; CHECK-NEXT: vadd.i32 q1, q1, r11
-; CHECK-NEXT: vadd.i32 q0, q0, r11
-; CHECK-NEXT: ldrb.w r10, [r5]
-; CHECK-NEXT: vmov r2, r5, d12
-; CHECK-NEXT: vadd.i32 q6, q5, r0
-; CHECK-NEXT: vadd.i32 q5, q4, r0
-; CHECK-NEXT: ldrb.w r1, [r8]
-; CHECK-NEXT: ldrb.w r9, [r4]
-; CHECK-NEXT: ldrb r4, [r6]
-; CHECK-NEXT: ldrb.w r6, [lr]
-; CHECK-NEXT: ldrb.w r12, [r12]
-; CHECK-NEXT: ldrb r2, [r2]
+; CHECK-NEXT: vadd.i32 q1, q5, r0
+; CHECK-NEXT: vadd.i32 q2, q4, r0
+; CHECK-NEXT: vmov r7, r3, d3
+; CHECK-NEXT: vadd.i32 q6, q0, lr
+; CHECK-NEXT: vmov r5, r6, d5
+; CHECK-NEXT: subs.w r9, r9, #16
+; CHECK-NEXT: vmov r4, r10, d2
+; CHECK-NEXT: vadd.i32 q1, q7, lr
+; CHECK-NEXT: vadd.i32 q4, q4, lr
+; CHECK-NEXT: vadd.i32 q5, q5, lr
+; CHECK-NEXT: ldrb.w r11, [r3]
+; CHECK-NEXT: ldrb r3, [r7]
+; CHECK-NEXT: vmov r7, r12, d4
+; CHECK-NEXT: vadd.i32 q2, q7, r0
+; CHECK-NEXT: vadd.i32 q7, q0, r0
; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: vmov.8 q4[0], r2
-; CHECK-NEXT: vmov.8 q4[1], r5
-; CHECK-NEXT: vmov r8, r5, d11
-; CHECK-NEXT: vmov.8 q4[2], r9
-; CHECK-NEXT: vmov.8 q4[3], r10
-; CHECK-NEXT: vmov.8 q4[4], r6
-; CHECK-NEXT: vmov.8 q4[5], r1
-; CHECK-NEXT: vmov.8 q4[6], r4
-; CHECK-NEXT: vmov r4, r6, d10
-; CHECK-NEXT: vmov.8 q4[7], r12
-; CHECK-NEXT: vmov q5, q3
-; CHECK-NEXT: ldrb.w lr, [r5]
-; CHECK-NEXT: vmov r5, r2, d13
-; CHECK-NEXT: ldrb r4, [r4]
; CHECK-NEXT: ldrb r6, [r6]
-; CHECK-NEXT: vmov.8 q4[8], r4
-; CHECK-NEXT: vmov.8 q4[9], r6
-; CHECK-NEXT: ldrb.w r9, [r2]
-; CHECK-NEXT: vmov r1, r2, d12
-; CHECK-NEXT: ldrb r5, [r5]
-; CHECK-NEXT: ldrb.w r10, [r2]
-; CHECK-NEXT: ldrb.w r2, [r8]
+; CHECK-NEXT: ldrb r4, [r4]
+; CHECK-NEXT: ldrb.w r10, [r10]
+; CHECK-NEXT: ldrb r7, [r7]
+; CHECK-NEXT: ldrb.w r1, [r12]
+; CHECK-NEXT: vmov.8 q0[0], r7
+; CHECK-NEXT: vmov.8 q0[1], r1
+; CHECK-NEXT: vmov r1, r7, d15
+; CHECK-NEXT: vmov.8 q0[2], r5
+; CHECK-NEXT: vmov.8 q0[3], r6
+; CHECK-NEXT: vmov.8 q0[4], r4
+; CHECK-NEXT: vmov r4, r2, d4
+; CHECK-NEXT: vmov.8 q0[5], r10
+; CHECK-NEXT: vmov.8 q0[6], r3
+; CHECK-NEXT: vmov.8 q0[7], r11
+; CHECK-NEXT: ldrb r6, [r7]
+; CHECK-NEXT: vmov r5, r7, d5
; CHECK-NEXT: ldrb r1, [r1]
-; CHECK-NEXT: vmov.8 q4[10], r2
-; CHECK-NEXT: vmov.8 q4[11], lr
-; CHECK-NEXT: vmov.8 q4[12], r1
-; CHECK-NEXT: vmov.8 q4[13], r10
-; CHECK-NEXT: vmov.8 q4[14], r5
-; CHECK-NEXT: vmov.8 q4[15], r9
-; CHECK-NEXT: vstrb.8 q4, [r7], #16
-; CHECK-NEXT: vmov q4, q2
+; CHECK-NEXT: ldrb r2, [r2]
+; CHECK-NEXT: ldrb r3, [r5]
+; CHECK-NEXT: ldrb.w r12, [r7]
+; CHECK-NEXT: ldrb r5, [r4]
+; CHECK-NEXT: vmov r4, r7, d14
+; CHECK-NEXT: vmov q7, q1
+; CHECK-NEXT: ldrb r4, [r4]
+; CHECK-NEXT: ldrb r7, [r7]
+; CHECK-NEXT: vmov.8 q0[8], r4
+; CHECK-NEXT: vmov.8 q0[9], r7
+; CHECK-NEXT: vmov.8 q0[10], r1
+; CHECK-NEXT: vmov.8 q0[11], r6
+; CHECK-NEXT: vmov.8 q0[12], r5
+; CHECK-NEXT: vmov.8 q0[13], r2
+; CHECK-NEXT: vmov.8 q0[14], r3
+; CHECK-NEXT: vmov.8 q0[15], r12
+; CHECK-NEXT: vstrb.8 q0, [r8], #16
+; CHECK-NEXT: vmov q0, q6
; CHECK-NEXT: bne .LBB17_3
; CHECK-NEXT: @ %bb.4: @ %middle.block
; CHECK-NEXT: @ in Loop: Header=BB17_2 Depth=1
-; CHECK-NEXT: ldr r3, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: ldr r1, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: cmp r3, r1
+; CHECK-NEXT: ldr.w r9, [sp, #52] @ 4-byte Reload
+; CHECK-NEXT: ldr r1, [sp, #60] @ 4-byte Reload
+; CHECK-NEXT: cmp r9, r1
; CHECK-NEXT: bne .LBB17_2
; CHECK-NEXT: .LBB17_5: @ %for.cond.cleanup
-; CHECK-NEXT: add sp, #16
-; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: add sp, #64
+; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .p2align 4
diff --git a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
index b548a6ee99412..eedca2cd4a5d3 100644
--- a/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-gather-scatter-optimisation.ll
@@ -602,57 +602,60 @@ define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr n
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT: .pad #4
; CHECK-NEXT: sub sp, #4
-; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13}
-; CHECK-NEXT: .pad #24
-; CHECK-NEXT: sub sp, #24
-; CHECK-NEXT: str r0, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT: .pad #32
+; CHECK-NEXT: sub sp, #32
+; CHECK-NEXT: strd r0, r2, [sp, #24] @ 8-byte Folded Spill
; CHECK-NEXT: cmp r3, #0
-; CHECK-NEXT: str r3, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT: str r3, [sp, #8] @ 4-byte Spill
; CHECK-NEXT: itt ne
-; CHECK-NEXT: ldrne r0, [sp, #112]
+; CHECK-NEXT: ldrne r0, [sp, #136]
; CHECK-NEXT: cmpne r0, #0
; CHECK-NEXT: bne .LBB10_2
; CHECK-NEXT: .LBB10_1: @ %for.cond.cleanup
-; CHECK-NEXT: add sp, #24
-; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13}
+; CHECK-NEXT: add sp, #32
+; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15}
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
; CHECK-NEXT: .LBB10_2: @ %for.cond1.preheader.us.preheader
-; CHECK-NEXT: ldr.w r12, [sp, #116]
+; CHECK-NEXT: ldr.w r12, [sp, #140]
; CHECK-NEXT: movs r7, #1
+; CHECK-NEXT: mov.w r11, #0
; CHECK-NEXT: vmov.i32 q0, #0x0
-; CHECK-NEXT: movs r5, #0
-; CHECK-NEXT: bic r0, r12, #3
-; CHECK-NEXT: subs r3, r0, #4
-; CHECK-NEXT: add.w r3, r7, r3, lsr #2
-; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill
-; CHECK-NEXT: ldr r3, [sp, #112]
-; CHECK-NEXT: lsl.w r7, r12, #1
-; CHECK-NEXT: str r7, [sp] @ 4-byte Spill
-; CHECK-NEXT: movs r7, #0
-; CHECK-NEXT: vdup.32 q1, r3
-; CHECK-NEXT: lsls r6, r3, #1
-; CHECK-NEXT: vshl.i32 q2, q1, #2
-; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT: str r3, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT: bic r2, r12, #3
+; CHECK-NEXT: subs r3, r2, #4
+; CHECK-NEXT: add.w r0, r7, r3, lsr #2
+; CHECK-NEXT: ldr r7, [sp, #136]
+; CHECK-NEXT: adr r3, .LCPI10_0
+; CHECK-NEXT: str r0, [sp, #16] @ 4-byte Spill
+; CHECK-NEXT: lsl.w r0, r12, #1
+; CHECK-NEXT: vdup.32 q1, r7
+; CHECK-NEXT: vldrw.u32 q2, [r3]
+; CHECK-NEXT: str r0, [sp, #4] @ 4-byte Spill
+; CHECK-NEXT: ldr r0, [sp, #24] @ 4-byte Reload
+; CHECK-NEXT: lsls r6, r7, #1
+; CHECK-NEXT: vshl.i32 q3, q1, #2
+; CHECK-NEXT: movs r3, #0
+; CHECK-NEXT: str r0, [sp, #20] @ 4-byte Spill
; CHECK-NEXT: b .LBB10_5
; CHECK-NEXT: .LBB10_3: @ %for.cond5.preheader.us73.preheader
; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT: add.w r3, r2, r8, lsl #1
+; CHECK-NEXT: ldr r0, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT: add.w r3, r0, r5, lsl #1
; CHECK-NEXT: wlstp.8 lr, r6, .LBB10_4
; CHECK-NEXT: b .LBB10_15
; CHECK-NEXT: .LBB10_4: @ %for.cond1.for.cond.cleanup3_crit_edge.us
; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT: ldr r3, [sp] @ 4-byte Reload
-; CHECK-NEXT: add r7, r12
-; CHECK-NEXT: ldr r5, [sp, #16] @ 4-byte Reload
-; CHECK-NEXT: add r5, r3
-; CHECK-NEXT: str r5, [sp, #16] @ 4-byte Spill
-; CHECK-NEXT: ldr r5, [sp, #8] @ 4-byte Reload
-; CHECK-NEXT: ldr r3, [sp, #4] @ 4-byte Reload
-; CHECK-NEXT: adds r5, #1
-; CHECK-NEXT: cmp r5, r3
+; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload
+; CHECK-NEXT: add r11, r12
+; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload
+; CHECK-NEXT: add r3, r0
+; CHECK-NEXT: str r3, [sp, #20] @ 4-byte Spill
+; CHECK-NEXT: ldr r3, [sp, #12] @ 4-byte Reload
+; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload
+; CHECK-NEXT: adds r3, #1
+; CHECK-NEXT: cmp r3, r0
; CHECK-NEXT: beq .LBB10_1
; CHECK-NEXT: .LBB10_5: @ %for.cond1.preheader.us
; CHECK-NEXT: @ =>This Loop Header: Depth=1
@@ -660,22 +663,21 @@ define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr n
; CHECK-NEXT: @ Child Loop BB10_11 Depth 3
; CHECK-NEXT: @ Child Loop BB10_14 Depth 3
; CHECK-NEXT: @ Child Loop BB10_15 Depth 2
-; CHECK-NEXT: ldr r3, [sp, #112]
+; CHECK-NEXT: mul r5, r3, r7
; CHECK-NEXT: cmp.w r12, #0
-; CHECK-NEXT: str r5, [sp, #8] @ 4-byte Spill
-; CHECK-NEXT: mul r8, r5, r3
+; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill
; CHECK-NEXT: beq .LBB10_3
; CHECK-NEXT: @ %bb.6: @ %for.cond5.preheader.us.us.preheader
; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1
-; CHECK-NEXT: mov.w r9, #0
+; CHECK-NEXT: mov.w r8, #0
; CHECK-NEXT: b .LBB10_8
; CHECK-NEXT: .LBB10_7: @ %for.cond5.for.cond.cleanup7_crit_edge.us.us
; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT: add.w r3, r9, r8
-; CHECK-NEXT: add.w r9, r9, #1
-; CHECK-NEXT: strh.w r10, [r2, r3, lsl #1]
-; CHECK-NEXT: ldr r3, [sp, #112]
-; CHECK-NEXT: cmp r9, r3
+; CHECK-NEXT: ldr r3, [sp, #28] @ 4-byte Reload
+; CHECK-NEXT: add.w r0, r8, r5
+; CHECK-NEXT: add.w r8, r8, #1
+; CHECK-NEXT: cmp r8, r7
+; CHECK-NEXT: strh.w r10, [r3, r0, lsl #1]
; CHECK-NEXT: beq .LBB10_4
; CHECK-NEXT: .LBB10_8: @ %for.cond5.preheader.us.us
; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1
@@ -690,48 +692,46 @@ define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr n
; CHECK-NEXT: b .LBB10_13
; CHECK-NEXT: .LBB10_10: @ %vector.ph
; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT: adr r3, .LCPI10_0
-; CHECK-NEXT: vmov q4, q1
-; CHECK-NEXT: vldrw.u32 q5, [r3]
-; CHECK-NEXT: ldr r3, [sp, #12] @ 4-byte Reload
-; CHECK-NEXT: vmov.i32 q3, #0x0
-; CHECK-NEXT: dls lr, r3
-; CHECK-NEXT: vmlas.i32 q4, q5, r9
-; CHECK-NEXT: ldr r3, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT: ldr r0, [sp, #16] @ 4-byte Reload
+; CHECK-NEXT: vmov q5, q1
+; CHECK-NEXT: vmov.i32 q4, #0x0
+; CHECK-NEXT: vmlas.i32 q5, q2, r8
+; CHECK-NEXT: dls lr, r0
+; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload
; CHECK-NEXT: .LBB10_11: @ %vector.body
; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1
; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2
; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
-; CHECK-NEXT: vadd.i32 q5, q4, q2
-; CHECK-NEXT: vldrh.s32 q6, [r1, q4, uxtw #1]
-; CHECK-NEXT: vldrh.s32 q4, [r3], #8
-; CHECK-NEXT: vmul.i32 q4, q6, q4
-; CHECK-NEXT: vadd.i32 q3, q4, q3
-; CHECK-NEXT: vmov q4, q5
+; CHECK-NEXT: vadd.i32 q6, q5, q3
+; CHECK-NEXT: vldrh.s32 q7, [r1, q5, uxtw #1]
+; CHECK-NEXT: vldrh.s32 q5, [r3], #8
+; CHECK-NEXT: vmul.i32 q5, q7, q5
+; CHECK-NEXT: vadd.i32 q4, q5, q4
+; CHECK-NEXT: vmov q5, q6
; CHECK-NEXT: le lr, .LBB10_11
; CHECK-NEXT: @ %bb.12: @ %middle.block
; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT: vaddv.u32 r10, q3
-; CHECK-NEXT: mov r4, r0
-; CHECK-NEXT: cmp r0, r12
+; CHECK-NEXT: vaddv.u32 r10, q4
+; CHECK-NEXT: mov r4, r2
+; CHECK-NEXT: cmp r2, r12
; CHECK-NEXT: beq .LBB10_7
; CHECK-NEXT: .LBB10_13: @ %for.body8.us.us.preheader
; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2
-; CHECK-NEXT: ldr r3, [sp, #112]
+; CHECK-NEXT: mla r3, r7, r4, r8
+; CHECK-NEXT: add.w r0, r11, r4
+; CHECK-NEXT: ldr r7, [sp, #24] @ 4-byte Reload
; CHECK-NEXT: sub.w lr, r12, r4
-; CHECK-NEXT: ldr r5, [sp, #20] @ 4-byte Reload
-; CHECK-NEXT: mla r3, r3, r4, r9
-; CHECK-NEXT: add.w r11, r1, r3, lsl #1
-; CHECK-NEXT: adds r3, r7, r4
-; CHECK-NEXT: add.w r3, r5, r3, lsl #1
+; CHECK-NEXT: add.w r9, r7, r0, lsl #1
+; CHECK-NEXT: ldr r7, [sp, #136]
+; CHECK-NEXT: add.w r3, r1, r3, lsl #1
; CHECK-NEXT: .LBB10_14: @ %for.body8.us.us
; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1
; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2
; CHECK-NEXT: @ => This Inner Loop Header: Depth=3
-; CHECK-NEXT: ldrsh.w r5, [r11]
-; CHECK-NEXT: add r11, r6
-; CHECK-NEXT: ldrsh r4, [r3], #2
-; CHECK-NEXT: smlabb r10, r5, r4, r10
+; CHECK-NEXT: ldrsh.w r4, [r3]
+; CHECK-NEXT: add r3, r6
+; CHECK-NEXT: ldrsh r0, [r9], #2
+; CHECK-NEXT: smlabb r10, r4, r0, r10
; CHECK-NEXT: le lr, .LBB10_14
; CHECK-NEXT: b .LBB10_7
; CHECK-NEXT: .LBB10_15: @ Parent Loop BB10_5 Depth=1
diff --git a/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll b/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll
index 8d3a4a632c863..7ecfca759d4d6 100644
--- a/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll
+++ b/llvm/test/CodeGen/X86/2011-09-14-valcoalesce.ll
@@ -27,12 +27,9 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
define void @BZ2_compressBlock() nounwind ssp {
; CHECK-LABEL: BZ2_compressBlock:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: pushl %ebx
-; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: subl $24, %esp
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_24
+; CHECK-NEXT: jne .LBB0_22
; CHECK-NEXT: # %bb.1: # %if.then68
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
@@ -41,12 +38,15 @@ define void @BZ2_compressBlock() nounwind ssp {
; CHECK-NEXT: .LBB0_3: # %for.cond19.preheader.i
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: xorl %ebx, %ebx
-; CHECK-NEXT: testb %bl, %bl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %al, %al
; CHECK-NEXT: jne .LBB0_5
; CHECK-NEXT: # %bb.4: # %if.then35.i
; CHECK-NEXT: .LBB0_5: # %if.end36.i
-; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $24, %esp
+; CHECK-NEXT: xorl %ebx, %ebx
; CHECK-NEXT: .LBB0_6: # %while.body.i188
; CHECK-NEXT: # =>This Loop Header: Depth=1
; CHECK-NEXT: # Child Loop BB0_8 Depth 2
@@ -54,7 +54,7 @@ define void @BZ2_compressBlock() nounwind ssp {
; CHECK-NEXT: movb $1, %dl
; CHECK-NEXT: testb %dl, %dl
; CHECK-NEXT: movl $0, %esi
-; CHECK-NEXT: jne .LBB0_14
+; CHECK-NEXT: jne .LBB0_12
; CHECK-NEXT: # %bb.7: # %while.body85.i.preheader
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: xorl %ecx, %ecx
@@ -69,64 +69,55 @@ define void @BZ2_compressBlock() nounwind ssp {
; CHECK-NEXT: jne .LBB0_10
; CHECK-NEXT: # %bb.9: # %while.body85.i
; CHECK-NEXT: # in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT: testb %bl, %bl
; CHECK-NEXT: jne .LBB0_8
; CHECK-NEXT: .LBB0_10: # %while.end.i
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: testb %bl, %bl
-; CHECK-NEXT: jne .LBB0_14
-; CHECK-NEXT: # %bb.11: # %land.lhs.true.i
-; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
-; CHECK-NEXT: testb %bl, %bl
-; CHECK-NEXT: jne .LBB0_14
-; CHECK-NEXT: # %bb.12: # %land.lhs.true103.i
-; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
-; CHECK-NEXT: testb %bl, %bl
-; CHECK-NEXT: jne .LBB0_14
-; CHECK-NEXT: # %bb.13: # %if.then108.i
+; CHECK-NEXT: jne .LBB0_12
+; CHECK-NEXT: # %bb.11: # %if.then108.i
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: xorl %esi, %esi
; CHECK-NEXT: movl %edx, %ecx
-; CHECK-NEXT: .LBB0_14: # %if.end117.i
+; CHECK-NEXT: .LBB0_12: # %if.end117.i
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: testb %bl, %bl
-; CHECK-NEXT: jne .LBB0_16
-; CHECK-NEXT: # %bb.15: # %if.then122.i
+; CHECK-NEXT: jne .LBB0_14
+; CHECK-NEXT: # %bb.13: # %if.then122.i
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; CHECK-NEXT: movl %esi, {{[0-9]+}}(%esp)
; CHECK-NEXT: movl %eax, {{[0-9]+}}(%esp)
; CHECK-NEXT: calll fprintf at PLT
-; CHECK-NEXT: .LBB0_16: # %for.cond138.preheader.i
+; CHECK-NEXT: .LBB0_14: # %for.cond138.preheader.i
; CHECK-NEXT: # in Loop: Header=BB0_6 Depth=1
; CHECK-NEXT: testb %bl, %bl
; CHECK-NEXT: incl %esi
; CHECK-NEXT: testb %bl, %bl
; CHECK-NEXT: movl %esi, %eax
; CHECK-NEXT: jne .LBB0_6
-; CHECK-NEXT: # %bb.17: # %for.cond182.preheader.i
+; CHECK-NEXT: # %bb.15: # %for.cond182.preheader.i
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_18
-; CHECK-NEXT: jmp .LBB0_19
-; CHECK-NEXT: .LBB0_18: # %for.inc220.us.i
-; CHECK-NEXT: .LBB0_19: # %while.body300.preheader.i
+; CHECK-NEXT: leal {{[0-9]+}}(%esp), %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: popl %ebx
+; CHECK-NEXT: jne .LBB0_16
+; CHECK-NEXT: jmp .LBB0_17
+; CHECK-NEXT: .LBB0_16: # %for.inc220.us.i
+; CHECK-NEXT: .LBB0_17: # %while.body300.preheader.i
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_21
-; CHECK-NEXT: .LBB0_20: # %for.end335.i
+; CHECK-NEXT: jne .LBB0_19
+; CHECK-NEXT: .LBB0_18: # %for.end335.i
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: jmp .LBB0_20
-; CHECK-NEXT: .LBB0_21: # %while.end2742.i
+; CHECK-NEXT: jmp .LBB0_18
+; CHECK-NEXT: .LBB0_19: # %while.end2742.i
; CHECK-NEXT: xorl %eax, %eax
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_23
-; CHECK-NEXT: # %bb.22: # %if.then2748.i
-; CHECK-NEXT: .LBB0_23: # %for.body2778.i
-; CHECK-NEXT: .LBB0_24: # %if.end85
-; CHECK-NEXT: addl $24, %esp
-; CHECK-NEXT: popl %esi
-; CHECK-NEXT: popl %ebx
+; CHECK-NEXT: jne .LBB0_21
+; CHECK-NEXT: # %bb.20: # %if.then2748.i
+; CHECK-NEXT: .LBB0_21: # %for.body2778.i
+; CHECK-NEXT: .LBB0_22: # %if.end85
; CHECK-NEXT: retl
entry:
br i1 undef, label %if.then68, label %if.end85
More information about the llvm-commits
mailing list